def start(self, debug=True, port=8000, address='127.0.0.1', app_settings={}): if debug: #import tornado.autoreload import logging logging.getLogger().setLevel(logging.DEBUG) address = '0.0.0.0' with NamedTemporaryFile(mode='w', suffix='.py', dir=os.path.abspath('.')) as _tempfile: if not getattr(self, 'settings_mod', False): if not getattr(self, 'settings_str', False): raise ("Could not configure biothings app") _tempfile.file.write(self.settings_str) _tempfile.file.flush() self.settings_mod = os.path.split( _tempfile.name)[1].split('.')[0] self.settings = BiothingESWebSettings(config=self.settings_mod) application = tornado.web.Application( self.settings.generate_app_list(), **app_settings) http_server = tornado.httpserver.HTTPServer(application) http_server.listen(port, address) loop = tornado.ioloop.IOLoop.instance() if debug: #tornado.autoreload.start(loop) logging.info('Server is running on "%s:%s"...' % (address, port)) loop.start()
def __init__(self, config_module=None): # About debug mode in tornado: # https://www.tornadoweb.org/en/stable/guide/running.html \ # #debug-mode-and-automatic-reloading logging.info("Biothings API %s", get_version()) self.config = BiothingESWebSettings(config_module) self.handlers = [] # additional handlers self.settings = dict(debug=False) self.host = None
def setup_class(cls): cls.settings = BiothingESWebSettings(cls.conf) prefix = cls.settings.API_PREFIX version = cls.settings.API_VERSION cls.path = f'/{prefix}/{version}/' cls.path = cls.path.replace('//', '/') cls.host = cls.host.rstrip('/')
""" import datetime import logging import os.path from tornado.ioloop import IOLoop from utils.api_monitor import update_uptime_status from utils.versioning import backup_and_refresh import config from biothings.web.index_base import main from biothings.web.settings import BiothingESWebSettings WEB_SETTINGS = BiothingESWebSettings(config=config) def schedule_daily_job(): tomorrow = datetime.datetime.today() + datetime.timedelta(days=1) midnight = datetime.datetime.combine(tomorrow, datetime.time.min) IOLoop.current().add_timeout(midnight.timestamp(), daily_job) def daily_job(): def sync_job(): backup_and_refresh() update_uptime_status() IOLoop.current().run_in_executor(None, sync_job) schedule_daily_job()
class BiothingsAPI(): """ Configure a Biothings Web API Server. There are three parts to it: * A biothings config module that defines the API handlers. * Additional Tornado handlers and application settings. * An asyncio event loop to run the tornado application. The API can be started with: * An external event loop by calling get_server() * A default tornado event loop by calling start() Unless started externally, debug mode: * Sets proper logging levels for root logger and es, * Enables debug mode on tornado except for autoreload, * Disables integrated tracking and error monitoring. """ def __init__(self, config_module=None): # About debug mode in tornado: # https://www.tornadoweb.org/en/stable/guide/running.html \ # #debug-mode-and-automatic-reloading logging.info("Biothings API %s", get_version()) self.config = BiothingESWebSettings(config_module) self.handlers = [] # additional handlers self.settings = dict(debug=False) self.host = None @staticmethod def use_curl(): """ Use curl implementation for tornado http clients. More on https://www.tornadoweb.org/en/stable/httpclient.html """ tornado.httpclient.AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient") def update(self, **settings): """ Update Tornado application settings. More on: https://www.tornadoweb.org/en/stable/web.html \ #tornado.web.Application.settings """ self.settings.update(settings) def _configure_logging(self): root_logger = logging.getLogger() self.config.configure_logger(root_logger) logging.getLogger('urllib3').setLevel(logging.ERROR) logging.getLogger('elasticsearch').setLevel(logging.WARNING) if self.settings['debug']: root_logger.setLevel(logging.DEBUG) es_tracer = logging.getLogger('elasticsearch.trace') es_tracer.setLevel(logging.DEBUG) es_tracer.addHandler(logging.NullHandler()) else: root_logger.setLevel(logging.INFO) def get_server(self): """ Run API in an external event loop. """ webapp = self.config.get_app(self.settings, self.handlers) server = tornado.httpserver.HTTPServer(webapp, xheaders=True) return server def start(self, port=8000): """ Run API in the default event loop. """ self._configure_logging() http_server = self.get_server() http_server.listen(port, self.host) logger = logging.getLogger('biothings.web') logger.info('Server is running on "%s:%s"...', self.host or '0.0.0.0', port) loop = tornado.ioloop.IOLoop.instance() loop.start()
class BiothingsTestCase(AsyncHTTPTestCase): ''' Starts a tornado server to run tests on. ''' settings = BiothingESWebSettings() host = os.getenv('TEST_HOST', f'/{settings.API_VERSION}').rstrip('/') # override def get_new_ioloop(self): return IOLoop.current() # override def get_app(self): app_list = self.settings.generate_app_list() settings = {"static_path": self.settings.STATIC_PATH} if getattr(self.settings, 'COOKIE_SECRET', None): settings["cookie_secret"] = self.settings.COOKIE_SECRET return Application(app_list, **settings) # override def request(self, path='/', method="GET", expect_status=200, **kwargs): ''' Make a requets with python requests library syntax. In addition, it compares response status code. ''' partial_func = partial(requests.request, method, self.get_url(path), **kwargs) res = self.io_loop.run_sync( lambda: self.io_loop.run_in_executor(None, partial_func), timeout=os.getenv("TEST_TIMEOUT")) assert res.status_code == expect_status return res # override def get_url(self, path): ''' Return the URL that can be passed to an HTTP client. When environment API_HOST is set to /v3: http://example.com/ -> http://example.com/ /query?q=cdk2 -> http://<test_server>/v3/query?q=cdk2 metadata -> http://<test_server>/v3/metadata When environment API_HOST is set to http://localhost:8000/api: http://example.com/ -> http://example.com/ /query?q=cdk2 -> http://localhost:8000/api/query?q=cdk2 metadata -> http://localhost:8000/api/metadata ''' if path.lower().startswith(("http://", "https://")): return path if not path.startswith('/'): return self.get_url('/' + path) if not path.startswith(self.host): return self.get_url(self.host + path) return super().get_url(path) def query(self, method='GET', endpoint='query', expect_hits=True, **kwargs): ''' Make a Biothings API query request. Query parameters are passed in as keyword arguments. ''' if method == 'GET': dic = self.request(endpoint, params=kwargs).json() if expect_hits: assert dic.get('hits', []), "No Hits" else: assert dic.get('hits', None) == [], f"Get {dic.get('hits')} instead." return dic if method == 'POST': lst = self.request(endpoint, method=method, data=kwargs).json() hits = False for item in lst: if "_id" in item: hits = True break if expect_hits: assert hits else: assert not hits return lst raise ValueError(f'Query method {method} is not supported.')
from biothings.web.index_base import main, options from biothings.web.settings import BiothingESWebSettings import os.path import config web_settings = BiothingESWebSettings(config=config) if __name__ == '__main__': (src_path, _) = os.path.split(os.path.abspath(__file__)) static_path = os.path.join(src_path, 'static') main(web_settings.generate_app_list(), app_settings={"cookie_secret": config.COOKIE_SECRET}, debug_settings={"static_path": static_path}, use_curl=True)
:param APP_LIST: a list of `URLSpec objects or (regex, handler_class) tuples <http://www.tornadoweb.org/en/stable/web.html#tornado.web.Application>`_ :param app_settings: `Tornado application settings <http://www.tornadoweb.org/en/stable/web.html#tornado.web.Application.settings>`_ :param debug_settings: Additional application settings for API debug mode :param sentry_client_key: Application-specific key for attaching Sentry monitor to the application :param use_curl: Overide the default simple_httpclient with curl_httpclient (Useful for Github Login) <https://www.tornadoweb.org/en/stable/httpclient.html> ''' settings = app_settings if options.debug: settings.update(debug_settings) settings.update({"debug": True}) application = get_app(APP_LIST, **settings) if __USE_SENTRY__ and sentry_client_key: application.sentry_client = AsyncSentryClient(sentry_client_key) if use_curl: tornado.httpclient.AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient") http_server = tornado.httpserver.HTTPServer(application) http_server.listen(options.port, address=options.address) loop = tornado.ioloop.IOLoop.instance() if options.debug: tornado.autoreload.start(loop) logging.info('Server is running on "%s:%s"...' % (options.address, options.port)) loop.start() if __name__ == '__main__': from biothings.web.settings import BiothingESWebSettings main(BiothingESWebSettings().generate_app_list())
def __new__(cls, *args, **kwargs): if not getattr(cls, 'settings', None): cls.settings = BiothingESWebSettings(config='config') return super(TornadoTestServerMixin, cls).__new__(cls)
def get_server(self, config_mod, **app_settings): settings = BiothingESWebSettings(config=config_mod) app = tornado.web.Application(settings.generate_app_list(), **app_settings) server = tornado.httpserver.HTTPServer(app) return server
class BiothingsAPIApp(object): def __init__(self, *args, **kwargs): if kwargs.get('object_name', False) or (len(args) >= 1 and not _is_module(args[0]) and not _is_file(args[0])): _arg = args[0] if args else '' self._configure_by_object_name( object_name=kwargs.get('object_name', _arg)) elif ((kwargs.get('config_file', False) and _file_exists(kwargs['config_file'])) or (len(args) >= 1 and _is_file(args[0]))): _arg = args[0] if args else '' self._configure_by_file( config_file=os.path.abspath(kwargs.get('config_file', _arg))) elif ((kwargs.get('config_module', False) and _is_module(kwargs['config_module'])) or (len(args) >= 1 and _is_module(args[0]))): _arg = args[0] if args else '' self._configure_by_module( config_module=kwargs.get('config_module', _arg)) else: self._configure_by_kwargs(**kwargs) def get_server(self, config_mod, **app_settings): settings = BiothingESWebSettings(config=config_mod) app = tornado.web.Application(settings.generate_app_list(), **app_settings) server = tornado.httpserver.HTTPServer(app) return server def start(self, debug=True, port=8000, address='127.0.0.1', app_settings={}): if debug: #import tornado.autoreload import logging logging.getLogger().setLevel(logging.DEBUG) address = '0.0.0.0' with NamedTemporaryFile(mode='w', suffix='.py', dir=os.path.abspath('.')) as _tempfile: if not getattr(self, 'settings_mod', False): if not getattr(self, 'settings_str', False): raise ("Could not configure biothings app") _tempfile.file.write(self.settings_str) _tempfile.file.flush() self.settings_mod = os.path.split( _tempfile.name)[1].split('.')[0] self.settings = BiothingESWebSettings(config=self.settings_mod) application = tornado.web.Application( self.settings.generate_app_list(), **app_settings) http_server = tornado.httpserver.HTTPServer(application) http_server.listen(port, address) loop = tornado.ioloop.IOLoop.instance() if debug: #tornado.autoreload.start(loop) logging.info('Server is running on "%s:%s"...' % (address, port)) loop.start() def _configure_by_object_name(self, object_name): # get the config file template config_string = """from biothings.web.settings.default import *\n""" + \ """from biothings.web.api.es.handlers import *\n""" + \ """ES_INDEX = '${src_package}_current'\n""" + \ """ES_DOC_TYPE = '${es_doctype}'\n""" + \ """API_VERSION = 'v1'\n""" + \ """APP_LIST = [(r'/status', StatusHandler), (r'/metadata/?', MetadataHandler), (r'/metadata/fields/?', MetadataHandler), (r'/{}/${annotation_endpoint}/(.+)/?'.format(API_VERSION), BiothingHandler), (r'/{}/${annotation_endpoint}/?$$'.format(API_VERSION), BiothingHandler), (r'/{}/query/?'.format(API_VERSION), QueryHandler), (r'/{}/metadata/?'.format(API_VERSION), MetadataHandler), (r'/{}/metadata/fields/?'.format(API_VERSION), MetadataHandler),]\n""" + \ """GA_RUN_IN_PROD = False""" settings_dict = { 'src_package': 'my' + object_name.lower(), 'es_doctype': object_name.lower(), 'annotation_endpoint': object_name.lower() } self.settings_str = Template(config_string).substitute(settings_dict) def _configure_by_module(self, config_module): self.settings_mod = config_module def _configure_by_kwargs(self, **kwargs): self.settings_str = """from biothings.web.settings.default import *\nfrom biothings.web.api.es.handlers import *\n""" for (k, v) in kwargs.items(): if k == 'APP_LIST': self.settings_str += '{k}=['.format(k=k) for (reg, handler_str) in v: self.settings_str += "(r'{reg}', {handler}),".format( reg=reg, handler=handler_str) self.settings_str += ']\n' elif k in [ 'ES_QUERY_BUILDER', 'ES_QUERY', 'ES_RESULT_TRANSFORMER' ]: self.settings_str += '{k}={v}\n' elif is_str(v): self.settings_str += '{k}="{v}"\n'.format(k=k, v=v) else: self.settings_str += '{k}={v}\n' def _configure_by_file(self, config_file): with open(config_file, 'r') as config_handle: self.settings_str = config_handle.read()