class PartialStore(ABC): """A store spawned inside partial-daemon container""" def __init__(self): self._logger = JinaLogger(self.__class__.__name__, **vars(jinad_args)) self.item = PartialStoreItem() self.object: Union[Type['BasePod'], Type['BaseDeployment'], 'Flow'] = None @abstractmethod def add(self, *args, **kwargs) -> PartialStoreItem: """Add a new element to the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" ... def delete(self) -> None: """Terminates the object in the store & stops the server""" try: if hasattr(self.object, 'close'): self.object.close() self._logger.info(self.item.arguments) if self.item.arguments.get('identity'): self._logger.success( f'{colored(self.item.arguments["identity"], "cyan")} is removed!' ) else: self._logger.success('object is removed!') else: self._logger.warning(f'nothing to close. exiting') except Exception as e: self._logger.error(f'{e!r}') raise else: self.item = PartialStoreItem()
def __init__(self, args: 'argparse.Namespace'): self.args = args if hasattr(self.args, 'port'): self.args.port = self.args.port self.args.parallel = self.args.shards self.name = self.args.name or self.__class__.__name__ self.is_forked = False self.logger = JinaLogger(self.name, **vars(self.args)) self._envs = {'JINA_DEPLOYMENT_NAME': self.name} if self.args.quiet: self._envs['JINA_LOG_CONFIG'] = 'QUIET' if self.args.env: self._envs.update(self.args.env) # arguments needed to create `runtime` and communicate with it in the `run` in the stack of the new process # or thread.f test_worker = multiprocessing.Process() self.is_ready = _get_event(test_worker) self.is_shutdown = _get_event(test_worker) self.cancel_event = _get_event(test_worker) self.is_started = _get_event(test_worker) self.ready_or_shutdown = ConditionalEvent( events_list=[self.is_ready, self.is_shutdown], ) self.runtime_ctrl_address = self._get_control_address() self._timeout_ctrl = self.args.timeout_ctrl
class PartialStore: """A store spawned inside mini-jinad container""" def __init__(self): self._logger = JinaLogger(self.__class__.__name__, **vars(jinad_args)) self.item = PartialStoreItem() def add(self, *args, **kwargs) -> PartialStoreItem: """Add a new element to the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" raise NotImplementedError def update(self, *args, **kwargs) -> PartialStoreItem: """Updates the element to the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" raise NotImplementedError def delete(self) -> None: """Terminates the object in the store & stops the server""" try: if hasattr(self, 'object'): self.object.close() else: self._logger.warning(f'nothing to close. exiting') except Exception as e: self._logger.error(f'{e!r}') raise
def __init__( self, metas: Optional[Dict] = None, requests: Optional[Dict] = None, runtime_args: Optional[Dict] = None, **kwargs, ): """`metas` and `requests` are always auto-filled with values from YAML config. :param metas: a dict of metas fields :param requests: a dict of endpoint-function mapping :param runtime_args: a dict of arguments injected from :class:`Runtime` during runtime :param kwargs: additional extra keyword arguments to avoid failing when extra params ara passed that are not expected """ self._add_metas(metas) self._add_requests(requests) self._add_runtime_args(runtime_args) self._init_monitoring() self.logger = JinaLogger(self.__class__.__name__) if __dry_run_endpoint__ not in self.requests: self.requests[__dry_run_endpoint__] = self._dry_run_func else: self.logger.warning( f' Endpoint {__dry_run_endpoint__} is defined by the Executor. Be aware that this endpoint is usually reserved to enable health checks from the Client through the gateway.' f' So it is recommended not to expose this endpoint. ')
class ExecMerger(Executor): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) from jina.logging.logger import JinaLogger self.logger = JinaLogger(self.__class__.__name__) @requests def debug(self, docs_matrix: List[DocumentArray], **kwargs): self.logger.debug( f'received doc matrix in exec-merger with length {len(docs_matrix)}.' ) result = DocumentArray() for docs in zip(*docs_matrix): traversed_executors = [ doc.tags['traversed-executors'] for doc in docs ] shard_ids = [doc.tags['shard_id'] for doc in docs] shards = [doc.tags['shards'] for doc in docs] parallels = [doc.tags['parallel'] for doc in docs] traversed_executors = list(chain(*traversed_executors)) doc = Document() doc.tags['traversed-executors'] = traversed_executors doc.tags['shard_id'] = shard_ids doc.tags['shards'] = shards doc.tags['parallel'] = parallels doc.tags['merged'] = True result.append(doc) return result
class PartialStore(ABC): """A store spawned inside partial-daemon container""" def __init__(self): self._logger = JinaLogger(self.__class__.__name__, **vars(jinad_args)) self.item = PartialStoreItem() self.object: Union['Pea', 'Pod', 'Flow'] = None @abstractmethod def add(self, *args, **kwargs) -> PartialStoreItem: """Add a new element to the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" ... def delete(self) -> None: """Terminates the object in the store & stops the server""" try: if hasattr(self.object, 'close'): self.object.close() else: self._logger.warning(f'nothing to close. exiting') except Exception as e: self._logger.error(f'{e!r}') raise
def _fetch_docker_auth(logger: JinaLogger) -> Tuple[str, str]: """Use Hub api to get docker credentials. :param logger: the logger instance :return: a dict of specifying username and password """ with open(os.path.join(__resources_path__, 'hubapi.yml')) as fp: hubapi_yml = JAML.load(fp) hubapi_url = hubapi_yml['hubapi']['url'] + hubapi_yml['hubapi'][ 'docker_auth'] with ImportExtensions( required=True, help_text= 'Missing "requests" dependency, please do pip install "jina[http]"', ): import requests headers = { 'Accept': 'application/json', 'authorizationToken': _fetch_access_token(logger), } response = requests.get(url=f'{hubapi_url}', headers=headers) if response.status_code != requests.codes.ok: raise HubLoginRequired( f'❌ Failed to fetch docker credentials. status code {response.status_code}' ) json_response = json.loads(response.text) username = base64.b64decode( json_response['docker_username']).decode('ascii') password = base64.b64decode( json_response['docker_password']).decode('ascii') logger.debug(f'✅ Successfully fetched docker creds for user') return username, password
def _register_to_mongodb(logger: JinaLogger, summary: Optional[Dict] = None): """Hub API Invocation to run `hub push`. :param logger: the logger instance :param summary: the summary dict object """ # TODO(Deepankar): implement to jsonschema based validation for summary logger.info('registering image to Jina Hub database...') with open(os.path.join(__resources_path__, 'hubapi.yml')) as fp: hubapi_yml = JAML.load(fp) hubapi_url = hubapi_yml['hubapi']['url'] + hubapi_yml['hubapi']['push'] with ImportExtensions( required=True, help_text= 'Missing "requests" dependency, please do pip install "jina[http]"', ): import requests headers = { 'Accept': 'application/json', 'authorizationToken': _fetch_access_token(logger), } response = requests.post(url=f'{hubapi_url}', headers=headers, data=json.dumps(summary)) if response.status_code == requests.codes.ok: logger.success(f'✅ Successfully updated the database. {response.text}') else: raise HubLoginRequired( f'❌ Got an error from the API: {response.text.rstrip()}. ' f'Please login using command: {colored("jina hub login", attrs=["bold"])}' )
def __call__( self, args: 'argparse.Namespace', is_started: Union['multiprocessing.Event', 'threading.Event'], is_shutdown: Union['multiprocessing.Event', 'threading.Event'], is_ready: Union['multiprocessing.Event', 'threading.Event'], is_cancelled: Union['multiprocessing.Event', 'threading.Event'], envs: Optional[Dict] = None, ): """Method responsible to manage a remote Pod This method is the target for the Pod's `thread` or `process` .. note:: Please note that env variables are process-specific. Subprocess inherits envs from the main process. But Subprocess's envs do NOT affect the main process. It does NOT mess up user local system envs. :param args: namespace args from the Pod :param is_started: concurrency event to communicate runtime is properly started. Used for better logging :param is_shutdown: concurrency event to communicate runtime is terminated :param is_ready: concurrency event to communicate runtime is ready to receive messages :param is_cancelled: concurrency event to receive cancelling signal from the Pod. Needed by some runtimes :param envs: a dictionary of environment variables to be passed to remote Pod """ self.args = args self.envs = envs self.is_started = is_started self.is_shutdown = is_shutdown self.is_ready = is_ready self.is_cancelled = is_cancelled self.pod_id = None self._logger = JinaLogger('RemotePod', **vars(args)) run_async(self._run)
def restart_deployment( name: str, namespace: str, image_name: str, container_cmd: str, container_args: str, logger: JinaLogger, replicas: int, pull_policy: str, custom_resource_dir: Optional[str] = None, port_expose: Optional[int] = None, ) -> str: """Restarts a service on Kubernetes. :param name: name of the service and deployment :param namespace: k8s namespace of the service and deployment :param image_name: image for the k8s deployment :param container_cmd: command executed on the k8s pods :param container_args: arguments used for the k8s pod :param logger: used logger :param replicas: number of replicas :param pull_policy: pull policy used for fetching the Docker images from the registry. :param custom_resource_dir: Path to a folder containing the kubernetes yml template files. Defaults to the standard location jina.resources if not specified. :param port_expose: port which will be exposed by the deployed containers :return: dns name of the created service """ # we can always assume the ports are the same for all executors since they run on different k8s pods # port expose can be defined by the user if not port_expose: port_expose = 8080 port_in = 8081 port_out = 8082 port_ctrl = 8083 logger.debug( f'🔋\tReplace Deployment for "{name}" with exposed port "{port_expose}"' ) kubernetes_tools.replace( deployment_name=name, namespace_name=namespace, template='deployment', params={ 'name': name, 'namespace': namespace, 'image': image_name, 'replicas': replicas, 'command': container_cmd, 'args': container_args, 'port_expose': port_expose, 'port_in': port_in, 'port_out': port_out, 'port_ctrl': port_ctrl, 'pull_policy': pull_policy, }, custom_resource_dir=custom_resource_dir, ) return f'{name}.{namespace}.svc'
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) from jina.logging.logger import JinaLogger self.logger = JinaLogger(self.__class__.__name__) self.logger.debug('Start sleep in SlowInitExecutor') time.sleep(10.0) self.logger.debug('Sleep over in SlowInitExecutor')
def __init__(self, id: 'DaemonID', files: List[UploadFile], name: str, *args, **kwargs) -> None: super().__init__(name=f'{self.__class__.__name__}{name}', daemon=True) self.id = id self.files = files self._logger = JinaLogger(self.name, workspace_path=self.workdir, **vars(jinad_args)) self.start()
def test_executor_image(logger: JinaLogger): image, build_logs = client.images.build( path=os.path.join(cur_dir, 'test-executor'), tag='test-executor:0.13.1' ) for chunk in build_logs: if 'stream' in chunk: for line in chunk['stream'].splitlines(): logger.debug(line) return image.tags[-1]
def dummy_dumper_image(logger: JinaLogger): image, build_logs = client.images.build( path=os.path.join(cur_dir, 'dummy-dumper'), tag='dummy-dumper:0.1.1' ) for chunk in build_logs: if 'stream' in chunk: for line in chunk['stream'].splitlines(): logger.debug(line) return image.tags[-1]
def __init__(self, **kwargs): super().__init__(**kwargs) self.logger = JinaLogger('CrudIndexer') self._docs = DocumentArray() self._dump_location = os.path.join(self.metas.workspace, 'docs') if os.path.exists(self._dump_location): self._docs = DocumentArray.load(self._dump_location) self.logger.info(f'Loaded {len(self._docs)} from {self._dump_location}') else: self.logger.info(f'No data found at {self._dump_location}')
class NameChangeExecutor(Executor): def __init__(self, runtime_args, *args, **kwargs): super().__init__(*args, **kwargs) self.name = runtime_args['name'] self.logger = JinaLogger(self.name) @requests def foo(self, docs: DocumentArray, **kwargs): self.logger.info(f'doc count {len(docs)}') docs.append(Document(text=self.name)) return docs
def build_docker_image(image_name, image_name_tag_map): logger = JinaLogger('kubernetes-testing') image_tag = image_name + ':' + image_name_tag_map[image_name] image, build_logs = client.images.build(path=os.path.join( cur_dir, image_name), tag=image_tag) for chunk in build_logs: if 'stream' in chunk: for line in chunk['stream'].splitlines(): logger.debug(line) return image.tags[-1]
def __init__( self, args: 'argparse.Namespace', **kwargs, ): super().__init__() self.args = args if args.name: self.name = f'{args.name}/{self.__class__.__name__}' else: self.name = self.__class__.__name__ self.logger = JinaLogger(self.name, **vars(self.args))
def __init__(self, dump_path: Optional[str] = None, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = JinaLogger('QueryExecutor') self._dump_path = dump_path or kwargs.get('runtime_args', {}).get( 'dump_path', None) if self._dump_path is not None and os.path.exists(self._dump_path): self.logger.success( f'loading Executor from dump path: {self._dump_path}') self._docs = DocumentArray.load(self._dump_path) else: self.logger.warning(f'no dump path passed. Loading an empty index') self._docs = DocumentArray()
def __init__(self, args: Optional[argparse.Namespace] = None, **kwargs): if args and isinstance(args, argparse.Namespace): self.args = args else: self.args = ArgNamespace.kwargs2namespace(kwargs, set_hub_parser()) self.logger = JinaLogger(self.__class__.__name__, **vars(args)) with ImportExtensions(required=True): import rich import cryptography import filelock assert rich #: prevent pycharm auto remove the above line assert cryptography assert filelock
def test_double_dynamic_routing_zmqstreamlet(): args1 = get_args() args2 = get_args() args3 = get_args() logger = JinaLogger('zmq-test') with ZmqStreamlet(args=args1, logger=logger) as z1, ZmqStreamlet( args=args2, logger=logger ) as z2, ZmqStreamlet(args=args3, logger=logger) as z3: assert z1.msg_sent == 0 assert z2.msg_sent == 0 assert z3.msg_sent == 0 req = jina_pb2.RequestProto() req.request_id = random_identity() d = req.data.docs.add() d.tags['id'] = 2 msg = Message(None, req, 'tmp', '') routing_pb = jina_pb2.RoutingTableProto() routing_table = { 'active_pod': 'executor1', 'pods': { 'executor1': { 'host': __default_host__, 'port': args1.port_in, 'expected_parts': 0, 'out_edges': [{'pod': 'executor2'}, {'pod': 'executor3'}], }, 'executor2': { 'host': __default_host__, 'port': args2.port_in, 'expected_parts': 1, 'out_edges': [], }, 'executor3': { 'host': __default_host__, 'port': args3.port_in, 'expected_parts': 1, 'out_edges': [], }, }, } json_format.ParseDict(routing_table, routing_pb) msg.envelope.routing_table.CopyFrom(routing_pb) for pea in [z1, z2, z3]: thread = threading.Thread(target=pea.start, args=(callback,)) thread.daemon = True thread.start() number_messages = 1000 for i in range(number_messages): z1.send_message(msg) time.sleep(5) assert z1.msg_sent == 2 * number_messages assert z1.msg_recv == 0 assert z2.msg_sent == 0 assert z2.msg_recv == number_messages assert z3.msg_sent == 0 assert z3.msg_recv == number_messages
def __init__( self, logger: Optional[JinaLogger] = None, compression: Optional[str] = None, metrics_registry: Optional['CollectorRegistry'] = None, ): self._logger = logger or JinaLogger(self.__class__.__name__) self.compression = (getattr(grpc.Compression, compression) if compression else grpc.Compression.NoCompression) if metrics_registry: with ImportExtensions( required=True, help_text= 'You need to install the `prometheus_client` to use the montitoring functionality of jina', ): from prometheus_client import Summary self._summary_time = Summary( 'sending_request_seconds', 'Time spent between sending a request to the Pod and receiving the response', registry=metrics_registry, namespace='jina', ).time() else: self._summary_time = contextlib.nullcontext() self._connections = self._ConnectionPoolMap(self._logger, self._summary_time) self._deployment_address_map = {}
def __init__(self, logger: Optional[JinaLogger] = None, on_demand_connection=True): self._connections = {} self._on_demand_connection = on_demand_connection self._logger = logger or JinaLogger(self.__class__.__name__)
def test_double_dynamic_routing_zmqlet(): args1 = get_args() args2 = get_args() args3 = get_args() logger = JinaLogger('zmq-test') with Zmqlet(args1, logger) as z1, Zmqlet(args2, logger) as z2, Zmqlet( args3, logger ) as z3: assert z1.msg_sent == 0 assert z2.msg_sent == 0 assert z3.msg_sent == 0 req = jina_pb2.RequestProto() req.request_id = random_identity() d = req.data.docs.add() d.tags['id'] = 2 msg = Message(None, req, 'tmp', '') routing_table = { 'active_pod': 'executor1', 'pods': { 'executor1': { 'host': __default_host__, 'port': args1.port_in, 'expected_parts': 0, 'out_edges': [{'pod': 'executor2'}, {'pod': 'executor3'}], }, 'executor2': { 'host': __default_host__, 'port': args2.port_in, 'expected_parts': 1, 'out_edges': [], }, 'executor3': { 'host': __default_host__, 'port': args3.port_in, 'expected_parts': 1, 'out_edges': [], }, }, } msg.envelope.routing_table.CopyFrom(RoutingTable(routing_table).proto) number_messages = 100 trips = 10 for i in range(trips): for j in range(number_messages): z1.send_message(msg) time.sleep(1) for i in range(number_messages): z2.recv_message(callback) z3.recv_message(callback) total_number_messages = number_messages * trips assert z1.msg_sent == 2 * total_number_messages assert z2.msg_sent == 0 assert z2.msg_recv == total_number_messages assert z3.msg_sent == 0 assert z3.msg_recv == total_number_messages
def __init__(self, hostname: str = '127.0.0.1', port: int = 5432, username: str = 'default_name', password: str = 'default_pwd', database: str = 'postgres', table: Optional[str] = 'default_table', *args, **kwargs): super().__init__(*args, **kwargs) self.logger = JinaLogger(self.__class__.__name__) self.hostname = hostname self.port = port self.username = username self.password = password self.database = database self.table = table
def __init__(self, dump_path: Optional[str] = None, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = JinaLogger('CompoundQueryExecutor') self._dump_path = dump_path if self._dump_path is not None and os.path.exists(self._dump_path): self._docs = DocumentArray.load(self._dump_path) else: self._docs = DocumentArray()
async def test_double_dynamic_routing_async_zmqlet(): args1 = get_args() args2 = get_args() args3 = get_args() logger = JinaLogger('zmq-test') with AsyncZmqlet(args1, logger) as z1, AsyncZmqlet( args2, logger) as z2, AsyncZmqlet(args3, logger) as z3: assert z1.msg_sent == 0 assert z2.msg_sent == 0 assert z3.msg_sent == 0 req = jina_pb2.RequestProto() req.request_id = random_identity() d = req.data.docs.add() d.tags['id'] = 2 msg = Message(None, req, 'tmp', '') routing_pb = jina_pb2.RoutingTableProto() routing_table = { 'active_pod': 'pod1', 'pods': { 'pod1': { 'host': '0.0.0.0', 'port': args1.port_in, 'expected_parts': 0, 'out_edges': [{ 'pod': 'pod2' }, { 'pod': 'pod3' }], }, 'pod2': { 'host': '0.0.0.0', 'port': args2.port_in, 'expected_parts': 1, 'out_edges': [], }, 'pod3': { 'host': '0.0.0.0', 'port': args3.port_in, 'expected_parts': 1, 'out_edges': [], }, }, } json_format.ParseDict(routing_table, routing_pb) msg.envelope.routing_table.CopyFrom(routing_pb) await send_msg(z1, msg) await z2.recv_message(callback) await z3.recv_message(callback) assert z1.msg_sent == 2 assert z1.msg_recv == 0 assert z2.msg_sent == 0 assert z2.msg_recv == 1 assert z3.msg_sent == 0 assert z3.msg_recv == 1
def test_color_log(): with JinaLogger('test_logger') as logger: logger.debug('this is test debug message') logger.info('this is test info message') logger.info(f'this is test {colored("color", "red")} message') logger.success('this is test success message') logger.warning('this is test warning message') logger.error('this is test error message') logger.critical('this is test critical message')
def test_logging_default(): with JinaLogger('test_logger') as logger: log(logger) try: import fluent assert len(logger.handlers) == 2 except (ModuleNotFoundError, ImportError): # if fluent not installed assert len(logger.handlers) == 2
def test_custom_swagger(p): args = set_gateway_parser().parse_args(p) logger = JinaLogger('') app = get_fastapi_app(args, logger) # The TestClient is needed here as a context manager to generate the shutdown event correctly # otherwise the app can hang as it is not cleaned up correctly # see https://fastapi.tiangolo.com/advanced/testing-events/ with TestClient(app) as client: assert any('/docs' in r.path for r in app.routes) assert any('/openapi.json' in r.path for r in app.routes)