def start(self,
debug=True,
port=8000,
address='127.0.0.1',
app_settings={}):
if debug:
#import tornado.autoreload
import logging
logging.getLogger().setLevel(logging.DEBUG)
address = '0.0.0.0'
with NamedTemporaryFile(mode='w',
suffix='.py',
dir=os.path.abspath('.')) as _tempfile:
if not getattr(self, 'settings_mod', False):
if not getattr(self, 'settings_str', False):
raise ("Could not configure biothings app")
_tempfile.file.write(self.settings_str)
_tempfile.file.flush()
self.settings_mod = os.path.split(
_tempfile.name)[1].split('.')[0]
self.settings = BiothingESWebSettings(config=self.settings_mod)
application = tornado.web.Application(
self.settings.generate_app_list(), **app_settings)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(port, address)
loop = tornado.ioloop.IOLoop.instance()
if debug:
#tornado.autoreload.start(loop)
logging.info('Server is running on "%s:%s"...' %
(address, port))
loop.start()
def __init__(self, config_module=None):
# About debug mode in tornado:
# https://www.tornadoweb.org/en/stable/guide/running.html \
# #debug-mode-and-automatic-reloading
logging.info("Biothings API %s", get_version())
self.config = BiothingESWebSettings(config_module)
self.handlers = [] # additional handlers
self.settings = dict(debug=False)
self.host = None
def setup_class(cls):
cls.settings = BiothingESWebSettings(cls.conf)
prefix = cls.settings.API_PREFIX
version = cls.settings.API_VERSION
cls.path = f'/{prefix}/{version}/'
cls.path = cls.path.replace('//', '/')
cls.host = cls.host.rstrip('/')
"""
import datetime
import logging
import os.path
from tornado.ioloop import IOLoop
from utils.api_monitor import update_uptime_status
from utils.versioning import backup_and_refresh
import config
from biothings.web.index_base import main
from biothings.web.settings import BiothingESWebSettings
WEB_SETTINGS = BiothingESWebSettings(config=config)
def schedule_daily_job():
tomorrow = datetime.datetime.today() + datetime.timedelta(days=1)
midnight = datetime.datetime.combine(tomorrow, datetime.time.min)
IOLoop.current().add_timeout(midnight.timestamp(), daily_job)
def daily_job():
def sync_job():
backup_and_refresh()
update_uptime_status()
IOLoop.current().run_in_executor(None, sync_job)
schedule_daily_job()
class BiothingsAPI():
"""
Configure a Biothings Web API Server.
There are three parts to it:
* A biothings config module that defines the API handlers.
* Additional Tornado handlers and application settings.
* An asyncio event loop to run the tornado application.
The API can be started with:
* An external event loop by calling get_server()
* A default tornado event loop by calling start()
Unless started externally, debug mode:
* Sets proper logging levels for root logger and es,
* Enables debug mode on tornado except for autoreload,
* Disables integrated tracking and error monitoring.
"""
def __init__(self, config_module=None):
# About debug mode in tornado:
# https://www.tornadoweb.org/en/stable/guide/running.html \
# #debug-mode-and-automatic-reloading
logging.info("Biothings API %s", get_version())
self.config = BiothingESWebSettings(config_module)
self.handlers = [] # additional handlers
self.settings = dict(debug=False)
self.host = None
@staticmethod
def use_curl():
"""
Use curl implementation for tornado http clients.
More on https://www.tornadoweb.org/en/stable/httpclient.html
"""
tornado.httpclient.AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient")
def update(self, **settings):
"""
Update Tornado application settings. More on:
https://www.tornadoweb.org/en/stable/web.html \
#tornado.web.Application.settings
"""
self.settings.update(settings)
def _configure_logging(self):
root_logger = logging.getLogger()
self.config.configure_logger(root_logger)
logging.getLogger('urllib3').setLevel(logging.ERROR)
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
if self.settings['debug']:
root_logger.setLevel(logging.DEBUG)
es_tracer = logging.getLogger('elasticsearch.trace')
es_tracer.setLevel(logging.DEBUG)
es_tracer.addHandler(logging.NullHandler())
else:
root_logger.setLevel(logging.INFO)
def get_server(self):
"""
Run API in an external event loop.
"""
webapp = self.config.get_app(self.settings, self.handlers)
server = tornado.httpserver.HTTPServer(webapp, xheaders=True)
return server
def start(self, port=8000):
"""
Run API in the default event loop.
"""
self._configure_logging()
http_server = self.get_server()
http_server.listen(port, self.host)
logger = logging.getLogger('biothings.web')
logger.info('Server is running on "%s:%s"...', self.host or '0.0.0.0',
port)
loop = tornado.ioloop.IOLoop.instance()
loop.start()
class BiothingsTestCase(AsyncHTTPTestCase):
'''
Starts a tornado server to run tests on.
'''
settings = BiothingESWebSettings()
host = os.getenv('TEST_HOST', f'/{settings.API_VERSION}').rstrip('/')
# override
def get_new_ioloop(self):
return IOLoop.current()
# override
def get_app(self):
app_list = self.settings.generate_app_list()
settings = {"static_path": self.settings.STATIC_PATH}
if getattr(self.settings, 'COOKIE_SECRET', None):
settings["cookie_secret"] = self.settings.COOKIE_SECRET
return Application(app_list, **settings)
# override
def request(self, path='/', method="GET", expect_status=200, **kwargs):
'''
Make a requets with python requests library syntax.
In addition, it compares response status code. '''
partial_func = partial(requests.request, method, self.get_url(path),
**kwargs)
res = self.io_loop.run_sync(
lambda: self.io_loop.run_in_executor(None, partial_func),
timeout=os.getenv("TEST_TIMEOUT"))
assert res.status_code == expect_status
return res
# override
def get_url(self, path):
'''
Return the URL that can be passed to an HTTP client.
When environment API_HOST is set to /v3:
http://example.com/ -> http://example.com/
/query?q=cdk2 -> http://<test_server>/v3/query?q=cdk2
metadata -> http://<test_server>/v3/metadata
When environment API_HOST is set to http://localhost:8000/api:
http://example.com/ -> http://example.com/
/query?q=cdk2 -> http://localhost:8000/api/query?q=cdk2
metadata -> http://localhost:8000/api/metadata
'''
if path.lower().startswith(("http://", "https://")):
return path
if not path.startswith('/'):
return self.get_url('/' + path)
if not path.startswith(self.host):
return self.get_url(self.host + path)
return super().get_url(path)
def query(self,
method='GET',
endpoint='query',
expect_hits=True,
**kwargs):
'''
Make a Biothings API query request.
Query parameters are passed in as keyword arguments. '''
if method == 'GET':
dic = self.request(endpoint, params=kwargs).json()
if expect_hits:
assert dic.get('hits', []), "No Hits"
else:
assert dic.get('hits',
None) == [], f"Get {dic.get('hits')} instead."
return dic
if method == 'POST':
lst = self.request(endpoint, method=method, data=kwargs).json()
hits = False
for item in lst:
if "_id" in item:
hits = True
break
if expect_hits:
assert hits
else:
assert not hits
return lst
raise ValueError(f'Query method {method} is not supported.')
from biothings.web.index_base import main, options
from biothings.web.settings import BiothingESWebSettings
import os.path
import config
web_settings = BiothingESWebSettings(config=config)
if __name__ == '__main__':
(src_path, _) = os.path.split(os.path.abspath(__file__))
static_path = os.path.join(src_path, 'static')
main(web_settings.generate_app_list(),
app_settings={"cookie_secret": config.COOKIE_SECRET},
debug_settings={"static_path": static_path},
use_curl=True)
:param APP_LIST: a list of `URLSpec objects or (regex, handler_class) tuples <http://www.tornadoweb.org/en/stable/web.html#tornado.web.Application>`_
:param app_settings: `Tornado application settings <http://www.tornadoweb.org/en/stable/web.html#tornado.web.Application.settings>`_
:param debug_settings: Additional application settings for API debug mode
:param sentry_client_key: Application-specific key for attaching Sentry monitor to the application
:param use_curl: Overide the default simple_httpclient with curl_httpclient (Useful for Github Login) <https://www.tornadoweb.org/en/stable/httpclient.html>
'''
settings = app_settings
if options.debug:
settings.update(debug_settings)
settings.update({"debug": True})
application = get_app(APP_LIST, **settings)
if __USE_SENTRY__ and sentry_client_key:
application.sentry_client = AsyncSentryClient(sentry_client_key)
if use_curl:
tornado.httpclient.AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient")
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(options.port, address=options.address)
loop = tornado.ioloop.IOLoop.instance()
if options.debug:
tornado.autoreload.start(loop)
logging.info('Server is running on "%s:%s"...' %
(options.address, options.port))
loop.start()
if __name__ == '__main__':
from biothings.web.settings import BiothingESWebSettings
main(BiothingESWebSettings().generate_app_list())
def __new__(cls, *args, **kwargs):
if not getattr(cls, 'settings', None):
cls.settings = BiothingESWebSettings(config='config')
return super(TornadoTestServerMixin, cls).__new__(cls)
def get_server(self, config_mod, **app_settings):
settings = BiothingESWebSettings(config=config_mod)
app = tornado.web.Application(settings.generate_app_list(),
**app_settings)
server = tornado.httpserver.HTTPServer(app)
return server
class BiothingsAPIApp(object):
def __init__(self, *args, **kwargs):
if kwargs.get('object_name',
False) or (len(args) >= 1 and not _is_module(args[0])
and not _is_file(args[0])):
_arg = args[0] if args else ''
self._configure_by_object_name(
object_name=kwargs.get('object_name', _arg))
elif ((kwargs.get('config_file', False)
and _file_exists(kwargs['config_file']))
or (len(args) >= 1 and _is_file(args[0]))):
_arg = args[0] if args else ''
self._configure_by_file(
config_file=os.path.abspath(kwargs.get('config_file', _arg)))
elif ((kwargs.get('config_module', False)
and _is_module(kwargs['config_module']))
or (len(args) >= 1 and _is_module(args[0]))):
_arg = args[0] if args else ''
self._configure_by_module(
config_module=kwargs.get('config_module', _arg))
else:
self._configure_by_kwargs(**kwargs)
def get_server(self, config_mod, **app_settings):
settings = BiothingESWebSettings(config=config_mod)
app = tornado.web.Application(settings.generate_app_list(),
**app_settings)
server = tornado.httpserver.HTTPServer(app)
return server
def start(self,
debug=True,
port=8000,
address='127.0.0.1',
app_settings={}):
if debug:
#import tornado.autoreload
import logging
logging.getLogger().setLevel(logging.DEBUG)
address = '0.0.0.0'
with NamedTemporaryFile(mode='w',
suffix='.py',
dir=os.path.abspath('.')) as _tempfile:
if not getattr(self, 'settings_mod', False):
if not getattr(self, 'settings_str', False):
raise ("Could not configure biothings app")
_tempfile.file.write(self.settings_str)
_tempfile.file.flush()
self.settings_mod = os.path.split(
_tempfile.name)[1].split('.')[0]
self.settings = BiothingESWebSettings(config=self.settings_mod)
application = tornado.web.Application(
self.settings.generate_app_list(), **app_settings)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(port, address)
loop = tornado.ioloop.IOLoop.instance()
if debug:
#tornado.autoreload.start(loop)
logging.info('Server is running on "%s:%s"...' %
(address, port))
loop.start()
def _configure_by_object_name(self, object_name):
# get the config file template
config_string = """from biothings.web.settings.default import *\n""" + \
"""from biothings.web.api.es.handlers import *\n""" + \
"""ES_INDEX = '${src_package}_current'\n""" + \
"""ES_DOC_TYPE = '${es_doctype}'\n""" + \
"""API_VERSION = 'v1'\n""" + \
"""APP_LIST = [(r'/status', StatusHandler), (r'/metadata/?', MetadataHandler), (r'/metadata/fields/?', MetadataHandler), (r'/{}/${annotation_endpoint}/(.+)/?'.format(API_VERSION), BiothingHandler), (r'/{}/${annotation_endpoint}/?$$'.format(API_VERSION), BiothingHandler), (r'/{}/query/?'.format(API_VERSION), QueryHandler), (r'/{}/metadata/?'.format(API_VERSION), MetadataHandler), (r'/{}/metadata/fields/?'.format(API_VERSION), MetadataHandler),]\n""" + \
"""GA_RUN_IN_PROD = False"""
settings_dict = {
'src_package': 'my' + object_name.lower(),
'es_doctype': object_name.lower(),
'annotation_endpoint': object_name.lower()
}
self.settings_str = Template(config_string).substitute(settings_dict)
def _configure_by_module(self, config_module):
self.settings_mod = config_module
def _configure_by_kwargs(self, **kwargs):
self.settings_str = """from biothings.web.settings.default import *\nfrom biothings.web.api.es.handlers import *\n"""
for (k, v) in kwargs.items():
if k == 'APP_LIST':
self.settings_str += '{k}=['.format(k=k)
for (reg, handler_str) in v:
self.settings_str += "(r'{reg}', {handler}),".format(
reg=reg, handler=handler_str)
self.settings_str += ']\n'
elif k in [
'ES_QUERY_BUILDER', 'ES_QUERY', 'ES_RESULT_TRANSFORMER'
]:
self.settings_str += '{k}={v}\n'
elif is_str(v):
self.settings_str += '{k}="{v}"\n'.format(k=k, v=v)
else:
self.settings_str += '{k}={v}\n'
def _configure_by_file(self, config_file):
with open(config_file, 'r') as config_handle:
self.settings_str = config_handle.read()