def __init__(self, args: Union['argparse.Namespace', Dict]):
super().__init__()
self.args = args
self.name = self.__class__.__name__ #: this is the process name
self.is_ready_event = _get_event(self)
self.is_shutdown = _get_event(self)
self.ready_or_shutdown = _make_or_event(self, self.is_ready_event,
self.is_shutdown)
self.is_shutdown.clear()
if 'daemon' in args:
self.daemon = args.daemon
if 'name' in self.args and self.args.name:
self.name = f'runtime-{self.args.name}'
if 'role' in self.args and self.args.role == PeaRoleType.PARALLEL:
self.name = f'runtime-{self.args.name}-{self.args.pea_id}'
if 'role' in self.args and self.args.role == PeaRoleType.HEAD:
self.name = f'runtime-{self.args.name}-head'
if 'role' in self.args and self.args.role == PeaRoleType.TAIL:
self.name = f'runtime-{self.args.name}-tail'
if 'host' in self.args and 'port_ctrl' in self.args and 'ctrl_with_ipc' in self.args:
self.ctrl_addr, self.ctrl_with_ipc = Zmqlet.get_ctrl_address(
self.args.host, self.args.port_ctrl, self.args.ctrl_with_ipc)
if 'log_id' in self.args and 'log_config' in self.args:
self.logger = JinaLogger(self.name,
log_id=self.args.log_id,
log_config=self.args.log_config)
else:
self.logger = JinaLogger(self.name)
def main(task, num_docs, top_k):
config()
workspace = os.environ["JINA_WORKSPACE"]
logger = JinaLogger('chinese-text-search')
if 'index' in task:
if os.path.exists(workspace):
logger.error(
f'\n +------------------------------------------------------------------------------------+ \
\n | 🤖🤖🤖 | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | 🤖🤖🤖 | \
\n +------------------------------------------------------------------------------------+'
)
sys.exit(1)
logger.info(f'### task = {task}')
if task == "index":
index(num_docs)
elif task == "index_restful":
index_restful(num_docs)
elif task == "query":
if not os.path.exists(workspace):
logger.warning(
f'The directory {workspace} does not exist. Please index first via `python app.py -t index`'
)
query(top_k)
elif task == "query_restful":
if not os.path.exists(workspace):
logger.warning(
f'The directory {workspace} does not exist. Please index first via `python app.py -t index`'
)
query_restful()
elif task == "dryrun":
dryrun()
def main(task, num_docs_query, num_docs_index):
config(task)
logger = JinaLogger('fashion-example-query')
workspace = os.environ['JINA_WORKDIR']
if task == 'index':
if os.path.exists(workspace):
logger.error(
f'\n +---------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +---------------------------------------------------------------------------------+'
)
sys.exit(1)
targets = download_fashionmnist()
if task == 'index':
index(num_docs_index, targets)
elif task == 'query':
if not os.path.exists(workspace):
logger.error(
f'The directory {workspace} does not exist. Please index first via `python app.py -t index`'
)
sys.exit(1)
query(num_docs_query, targets)
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `query`.'
)
def main(task, num_docs, request_size, data_set, model_name):
config(model_name)
workspace = os.environ['JINA_WORKSPACE']
logger = JinaLogger('cross-modal-search')
if 'index' in task:
if os.path.exists(workspace):
logger.error(
f'\n +------------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +------------------------------------------------------------------------------------+'
)
sys.exit(1)
logger.info(f'### task = {task}')
if task == 'index':
with Flow.load_config('flow-index.yml') as f:
with TimeContext(f'QPS: indexing {num_docs}', logger=f.logger):
f.index(
input_fn=input_index_data(num_docs, request_size, data_set),
request_size=request_size
)
elif task == 'index_restful':
index_restful(num_docs)
elif task == 'query':
with Flow.load_config('flow-query.yml') as f:
f.use_rest_gateway()
f.block()
elif task == 'query_restful':
if not os.path.exists(workspace):
logger.warning(f'The directory {workspace} does not exist. Please index first via `python app.py -t index`')
query_restful()
elif task == 'dryrun':
dryrun()
def main(task, num_docs_index):
config()
workspace = os.environ["JINA_WORKSPACE"]
logger = JinaLogger('multires-lyrics-search')
if task == 'index':
config()
workspace = os.environ['JINA_WORKDIR']
if os.path.exists(workspace):
logger.warning(
f'\n +---------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +---------------------------------------------------------------------------------+'
)
index()
elif task == 'index_restful':
index_restful(num_docs_index)
elif task == 'query':
config()
query()
elif task == 'query_restful':
if not os.path.exists(workspace):
logger.warning(
f'The directory {workspace} does not exist. Please index first via `python app.py -t index`'
)
query_restful()
elif task == 'dryrun':
dryrun()
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `query`.'
)
def test_register_to_mongodb(dummy_access_token, tmpdir, monkeypatch, mocker, summary, response_code):
import json
from pathlib import Path
class MockResponse:
def __init__(self, content):
self.content = content
def json(self):
return self.content
def status_code(self):
return response_code
mock = mocker.Mock()
def _mock_post(url, headers, data):
mock(url=url, headers=headers, data=data)
resp = {'text': f'return status code {response_code}'}
return MockResponse(resp)
def _mock_home():
return Path(str(tmpdir))
monkeypatch.setattr(requests, 'post', _mock_post)
monkeypatch.setattr(Path, 'home', _mock_home)
_register_to_mongodb(JinaLogger('test_mongodb_register'), summary)
request_args = mock.call_args_list[0][1]
assert request_args['url'] == 'https://hubapi.jina.ai/push'
assert request_args['headers'] == {'Accept': 'application/json', 'authorizationToken': f'{DUMMY_ACCESS_TOKEN}'}
assert request_args['data'] == json.dumps(summary)
def test_logging_default():
with JinaLogger('test_logger') as logger:
log(logger)
try:
import fluent
assert len(logger.handlers) == 3
except (ModuleNotFoundError, ImportError):
# if fluent not installed
assert len(logger.handlers) == 2
def test_hub_build_level_fail(monkeypatch, test_workspace, docker_image):
args = set_hub_build_parser().parse_args(['path/hub-mwu', '--push', '--host-info', '--test-level', 'FLOW'])
expected_failed_levels = [BuildTestLevel.POD_NONDOCKER, BuildTestLevel.POD_DOCKER, BuildTestLevel.FLOW]
_, failed_levels = HubIO(args)._test_build(docker_image, BuildTestLevel.FLOW,
os.path.join(cur_dir, 'yaml/test-joint.yml'), 60, True,
JinaLogger('unittest'))
assert expected_failed_levels == failed_levels
def test_logging_file():
fn = f'jina-{__uptime__}.log'
if os.path.exists(fn):
os.remove(fn)
with JinaLogger('test_logger', log_config=os.path.join(cur_dir, 'yaml/file.yml')) as logger:
log(logger)
assert os.path.exists(fn)
with open(fn) as fp:
assert len(fp.readlines()) == 7
os.remove(fn)
def test_logging_file():
fn = f'jina-{__uptime__}.log'
if Path(fn).exists():
os.remove(fn)
with JinaLogger('test_logger', log_config=str(cur_dir / 'yaml' / 'file.yml')) as logger:
log(logger)
assert Path(fn).exists()
with open(fn) as fp:
assert len(fp.readlines()) == 7
os.remove(fn)
def test_hub_build_level_pass(monkeypatch, test_workspace):
args = set_hub_build_parser().parse_args(
['path/hub-mwu', '--push', '--host-info', '--test-level', 'EXECUTOR'])
docker_image = cli.get_image('jinahub/pod.dummy_mwu_encoder')
expected_failed_levels = []
_, failed_levels = HubIO(args)._test_build(
docker_image, BuildTestLevel.EXECUTOR,
os.path.join(cur_dir, 'yaml/test-joint.yml'), 60, True,
JinaLogger('unittest'))
assert expected_failed_levels == failed_levels
def test_logging_fluentd(monkeypatch, log_config):
from fluent import asynchandler as fluentasynchandler
with JinaLogger('test_logger', log_config=log_config, log_id='test_log_id') as logger:
def mock_emit(obj, record):
msg = obj.format(record)
assert msg['log_id'] == 'test_log_id'
assert msg['context'] == 'test_logger'
assert msg['name'] == 'test_logger'
assert msg['type'] == 'INFO'
assert msg['message'] == 'logging progress'
monkeypatch.setattr(fluentasynchandler.FluentHandler, "emit", mock_emit)
logger.info('logging progress')
def test_hubapi_list(mocker):
mocker.return_value.__enter__.return_value.read.return_value = json.dumps(
sample_manifest)
result = remote._list(
logger=JinaLogger('list'),
image_name='Dummy MWU Encoder',
image_kind='encoder',
image_type='pod',
image_keywords=['toy'],
)
mocker.assert_called_once()
assert result[0]['name'] == 'Dummy MWU Encoder'
assert result[0]['version'] == '0.0.52'
assert result[0]['kind'] == 'encoder'
def test_docker_auth_failure(mocker, dummy_access_token, monkeypatch, tmpdir):
mock = mocker.Mock()
def _mock_get(url, headers):
mock(url=url, headers=headers)
return MockResponse(response_code=requests.codes.unauthorized)
def _mock_home():
return Path(str(tmpdir))
monkeypatch.setattr(requests, 'get', _mock_get)
monkeypatch.setattr(Path, 'home', _mock_home)
with pytest.raises(HubLoginRequired):
_fetch_docker_auth(logger=JinaLogger('test_docker_auth'))
def test_register_to_mongodb_failure(dummy_access_token, tmpdir, monkeypatch,
mocker, summary, response_code):
mock = mocker.Mock()
def _mock_post(url, headers, data):
mock(url=url, headers=headers, data=data)
return MockResponse(response_code=response_code)
def _mock_home():
return Path(str(tmpdir))
monkeypatch.setattr(requests, 'post', _mock_post)
monkeypatch.setattr(Path, 'home', _mock_home)
with pytest.raises(HubLoginRequired):
_register_to_mongodb(JinaLogger('test_mongodb_register'), summary)
def test_hub_build_level_pass(monkeypatch, test_workspace, docker_image):
args = set_hub_build_parser().parse_args(
['path/hub-mwu', '--push', '--host-info', '--test-level', 'EXECUTOR']
)
expected_failed_levels = []
_, failed_levels = HubIO(args)._test_build(
docker_image,
BuildTestLevel.EXECUTOR,
os.path.join(cur_dir, 'yaml/test-executor.yml'),
60000,
True,
JinaLogger('unittest'),
)
assert expected_failed_levels == failed_levels
def test_logging_fluentd(monkeypatch, log_config):
from fluent import asynchandler as fluentasynchandler
with JinaLogger('test_logger', log_config=log_config,
identity='test_log_id', workspace_path='/tmp/test/') as logger:
def mock_emit(obj, record):
msg = obj.format(record)
assert msg['workspace_path'] == '/tmp/test/'
assert msg['log_id'] == 'test_log_id'
assert msg['context'] == 'test_logger'
assert msg['name'] == 'test_logger'
assert msg['type'] == 'INFO'
assert msg['message'] == 'logging progress'
datetime.fromisoformat(msg['uptime'])
monkeypatch.setattr(fluentasynchandler.FluentHandler, 'emit', mock_emit)
logger.info('logging progress')
def test_docker_auth_success(mocker, dummy_access_token, monkeypatch, tmpdir):
mock = mocker.Mock()
def _mock_get(url, headers):
mock(url=url, headers=headers)
return MockResponse(response_code=requests.codes.ok)
def _mock_home():
return Path(str(tmpdir))
monkeypatch.setattr(requests, 'get', _mock_get)
monkeypatch.setattr(Path, 'home', _mock_home)
username, password = _fetch_docker_auth(
logger=JinaLogger('test_docker_auth'))
assert username == 'abc'
assert password == 'def'
class InMemoryStore:
_store = {}
# TODO: Implement fastapi based oauth/bearer security here
credentials = 'foo:bar'
_session_token = None
logger = JinaLogger(context='🏪 STORE')
@contextmanager
def _session(self):
if self._session_token:
yield
return
self._session_token = self._login(self.credentials)
try:
yield
finally:
self._logout(self._session_token)
# TODO: implement login-logout here to manage session token
def _login(self, creds):
token = hash(creds)
self.logger.debug(f'LOGIN: {token}')
return token
def _logout(self, token):
self.logger.debug(f'LOGOUT: {token}')
def _create(self, **kwargs):
raise NotImplementedError
def _start(self, context):
return context.start()
def _close(self, context):
context.close()
def _delete(self, id: uuid.UUID):
raise NotImplementedError
def _delete_all(self):
for _id in self._store.copy().keys():
self._delete(_id)
def test_register_to_mongodb_success(dummy_access_token, tmpdir, monkeypatch,
mocker, summary):
mock = mocker.Mock()
def _mock_post(url, headers, data):
mock(url=url, headers=headers, data=data)
return MockResponse(response_code=requests.codes.ok)
def _mock_home():
return Path(str(tmpdir))
monkeypatch.setattr(requests, 'post', _mock_post)
monkeypatch.setattr(Path, 'home', _mock_home)
_register_to_mongodb(JinaLogger('test_mongodb_register'), summary)
request_args = mock.call_args_list[0][1]
assert request_args['url'] == 'https://hubapi.jina.ai/push'
assert request_args['headers'] == {
'Accept': 'application/json',
'authorizationToken': f'{DUMMY_ACCESS_TOKEN}',
}
assert request_args['data'] == json.dumps(summary)
def main(task, num_docs, request_size, data_set, model_name):
config(model_name)
workspace = os.environ['JINA_WORKSPACE']
logger = JinaLogger('cross-modal-search')
if 'index' in task:
if os.path.exists(workspace):
logger.error(
f'\n +------------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +------------------------------------------------------------------------------------+'
)
sys.exit(1)
if 'query' in task and not os.path.exists(workspace):
logger.info(f"The directory {workspace} does not exist. Please index first via `python app.py -t index`")
sys.exit(1)
logger.info(f'### task = {task}')
if task == 'index':
index(data_set, num_docs, request_size)
if task == 'query_restful':
query_restful()
def test_logging_file():
with JinaLogger('test_logger', log_config='yaml/file.yml') as logger:
log(logger)
assert os.path.exists(f'jina-{__uptime__}.log')
with open(f'jina-{__uptime__}.log') as fp:
assert len(fp.readlines()) == 5
def test_logging_level_os_environ_variable(config):
with JinaLogger('test_logger', log_config=os.path.join(cur_dir, 'yaml/file.yml')) as logger:
log(logger)
assert logger.logger.level == LogVerbosity.from_string('SUCCESS')
import json
import subprocess
import threading
import pkg_resources
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from uvicorn import Config, Server
from daemon.excepts import Runtime400Exception, daemon_runtime_exception_handler
from jina import __version__
from jina.logging import JinaLogger
from .parser import get_main_parser, _get_run_args
jinad_args = get_main_parser().parse_args([])
daemon_logger = JinaLogger('DAEMON', **vars(jinad_args))
def _get_app():
from .api.endpoints import router, flow, pod, pea, logs, workspace
app = FastAPI(
title='JinaD (Daemon)',
description='REST interface for managing distributed Jina',
version=__version__,
openapi_tags=[{
'name': 'daemon',
'description': 'API to manage the Daemon',
}, {
'name': 'flows',
'description': 'API to manage Flows',
}, {
import json
import os
import subprocess
import threading
from collections import namedtuple
import pkg_resources
from fastapi import FastAPI
from uvicorn import Config, Server
from jina.logging import JinaLogger
from .parser import get_main_parser
daemon_logger = JinaLogger(context='👻',
log_config=os.getenv(
'JINAD_LOG_CONFIG',
pkg_resources.resource_filename(
'jina', '/'.join(
('resources', 'logging.daemon.yml')))))
def _get_app():
from .api.endpoints import common_router, flow, pod, pea, logs
from .config import jinad_config, fastapi_config, openapitags_config
context = namedtuple('context', ['router', 'openapi_tags', 'tags'])
_all_routers = {
'flow':
context(router=flow.router,
openapi_tags=openapitags_config.FLOW_API_TAGS,
tags=[openapitags_config.FLOW_API_TAGS[0]['name']]),
'pod':
import os
import shutil
import sys
from glob import glob
import click
from jina.flow import Flow
from jina import Document
from jina.logging.profile import TimeContext
from jina.logging import JinaLogger
MAX_DOCS = os.environ.get('MAX_DOCS', 16)
BATCH_SIZE = 16
logger = JinaLogger('object-search')
def config():
os.environ['JINA_DATA_FILE'] = os.environ.get('JINA_DATA_FILE',
'data/**/*.jpg')
os.environ['PARALLEL'] = '1'
os.environ['SHARDS'] = '1'
os.environ['JINA_PORT'] = os.environ.get('JINA_PORT', str(45678))
os.environ['WORKDIR'] = os.environ.get('JINA_WORKDIR', './workspace')
def index(data_path, batch_size, num_docs: int):
f = Flow.load_config('flow-index.yml')
num_docs = min(num_docs, len(glob(data_path)))
with f:
import json
import uuid
from typing import List, Union
from fastapi import status, APIRouter, Body, Response, File, UploadFile
from jina.parser import set_client_cli_parser
from jina.helper import get_parsed_args
from jina.logging import JinaLogger
from jina.clients import Client
from jinad.store import flow_store
from jinad.models.pod import PodModel
from jinad.excepts import FlowYamlParseException, FlowCreationException, FlowStartException, \
HTTPException
logger = JinaLogger(context='👻 FLOWAPI')
router = APIRouter()
@router.put(
path='/flow/pods',
summary='Build & start a Flow using Pods',
)
async def _create_from_pods(
pods: Union[List[PodModel]] = Body(...,
example=json.loads(PodModel().json()))
):
"""
Build a Flow using a list of `PodModel`
[
def test_logging_syslog():
with JinaLogger('test_logger', log_config=os.path.join(cur_dir, 'yaml/syslog.yml')) as logger:
log(logger)
assert len(logger.handlers) == 1
def test_logging_level_yaml():
with JinaLogger('test_logger', log_config=os.path.join(cur_dir, 'yaml/file.yml')) as logger:
log(logger)
assert logger.logger.level == LogVerbosity.from_string('INFO')
def test_logging_syslog():
with JinaLogger('test_logger', log_config=str(cur_dir / 'yaml' / 'syslog.yml')) as logger:
log(logger)
assert len(logger.handlers) == 1