def tcount_exception(counter_name): exc_name, exc_value, lineno, lines = get_exception_info() data = generate_json_record(counter_name, exc_name, exc_value, lineno, lines) # configure to use the better fetch method AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") # make the actual call to the counter AsyncHTTPClient().fetch(URL, lambda resp: None, method="POST", body=json.dumps(data))
def main(): term, location, search_limit = 'dinner', 'Chattanooga, TN', 3 url_params = { 'term': term.replace(' ', '+'), 'location': location.replace(' ','+'), 'limit' : search_limit } def on_response(response, error): ioloop.IOLoop.instance().stop() if error: raise error for repo in response: print repo auth = OAuth1(client_key=CLIENT_KEY, client_secret=CLIENT_SECRET, resource_owner_key=RESOURCE_OWNER_KEY, resource_owner_secret=RESOURCE_OWNER_SECRET, ) AsyncHTTPClient.configure(None, defaults=dict(user_agent="ipa-agent")) buss = Businesses(AsyncHTTPClient()) buss.all(on_response, auth=auth, url_params=url_params) ioloop.IOLoop.instance().start()
def getAClient(max_clients=200): client = AsyncHTTPClient(max_clients=max_clients) # page 20 * size 20,maybe great client.configure(None, # "tornado.curl_httpclient.CurlAsyncHTTPClient", raise_error=False ) return client
def http_request(url, server, port=80, timeout=20.0): def check_twitter_response(response): return (response is not None and len(response.headers.get_list('Server')) > 0 \ and response.headers.get_list('Server')[0] == 'tfe') def get_rate_limits(headers): rtime = headers.get_list('X-RateLimit-Reset') rhits = headers.get_list('X-RateLimit-Remaining') if len(rtime) > 0: rtime = int(rtime[0]) else: rtime = None if len(rhits) > 0: rhits = int(rhits[0]) else: rhits = None return rhits, rtime AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") http_client = HTTPClient() code, data, rtime, rhits = 999, None, None, None try: response = http_client.fetch(url, proxy_host=server, proxy_port=port, connect_timeout=timeout, request_timeout=timeout) response.rethrow() if check_twitter_response(response): code, data = response.code, response.body rhits, rtime = get_rate_limits(response.headers) except HTTPError as e: if check_twitter_response(e.response): code, data = e.code, None rhits, rtime = get_rate_limits(e.response.headers) return code, data, rhits, rtime
def __init__(self,url,action,target,callback=None,retry=0,ssl=False): """ Build the event source client :param url: string, the url to connect to :param action: string of the listening action to connect to :param target: string with the listening token :param callback: function with one parameter (Event) that gets called for each received event :param retry: timeout between two reconnections (0 means no reconnection) """ log.debug("EventSourceClient(%s,%s,%s,%s,%s)" % (url,action,target,callback,retry)) if ssl: self._url = "https://%s/%s/%s" % (url,action,target) else: self._url = "http://%s/%s/%s" % (url,action,target) AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") self.http_client = AsyncHTTPClient() self.http_request = HTTPRequest(url=self._url, method='GET', headers={"content-type":"text/event-stream"}, request_timeout=0, streaming_callback=self.handle_stream) if callback is None: self.cb = lambda e: log.info( "received %s" % (e,) ) else: self.cb = callback self.retry_timeout = int(retry)
def __init__(self, url, callback = None, retry = 0, keep_alive = False, ssl = False, validate_cert = False, user = None, password = None, async_http_client = DEFAULT_ASYNC_HTTP_CLIENT): """ Build the event source client :param url: string, the url to connect to :param action: string of the listening action to connect to :param target: string with the listening token :param callback: function with one parameter (Event) that gets called for each received event :param retry: timeout between two reconnections (0 means no reconnection) """ log.debug("EventSourceClient(%s,%s,%s)" % (url, callback, retry)) self.data_partial = None self.last_event_id = None self.retry_timeout = int(retry) self.keep_alive = keep_alive self.current_event = None self._url = url self._headers = {"Accept": "text/event-stream"} self._options = { 'validate_cert': validate_cert, 'auth_username': user, 'auth_password': password } if async_http_client is not None: AsyncHTTPClient.configure(async_http_client) self.http_client = AsyncHTTPClient() if callback is None: self.cb = lambda e: log.info( "received %s" % (e,) ) else: self.cb = callback
def __init__(self, host = 'localhost', max_clients=100, port = 4711, logger = None): AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") self.http_client = AsyncHTTPClient(max_clients=max_clients) self.host = host self.port = port self.logger = logger self.body = ''
def __init__(self): AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient") self.client = AsyncHTTPClient() # 开发者 token self.client_id = WEI_CLIENT_ID self.client_secret = WEI_CLIENT_SECRET self.admin_token = WEI_ADMIN_TOKEN # oauth 相关 url self.oauth2_url = 'https://api.weibo.com/oauth2/authorize?' \ 'client_id=%s&' \ 'response_type=code&' \ 'redirect_uri=twiwei.com/oauth2/weibo/access_token/' % self.client_id self.access_token_url = 'https://api.weibo.com/oauth2/access_token' # 时间线相关 url self.user_home_url = 'https://api.weibo.com/2/statuses/home_timeline.json' self.public_url = 'https://api.weibo.com/2/statuses/public_timeline.json' # 用户操作相关 url self.like_weibo_url = 'https://api.weibo.com/2/favorites/create.json' self.unlike_weibo_url = 'https://api.weibo.com/2/favorites/destroy.json' self.get_replies_url = 'https://api.weibo.com/2/comments/show.json' self.repost_message_url = 'https://api.weibo.com/2/statuses/repost.json' self.reply_message_url = 'https://api.weibo.com/2/comments/create.json' # 用户信息相关 url self.user_info_url = 'https://api.weibo.com/2/users/show.json' self.liked_url = 'https://api.weibo.com/2/favorites/ids.json'
def __init__( self, concurrency=10, auto_start=False, cache=False, expiration_in_seconds=30, request_timeout_in_seconds=10, connect_timeout_in_seconds=5, ignore_pycurl=False, limiter=None, allow_connection_reuse=True): self.concurrency = concurrency self.auto_start = auto_start self.last_timeout = None self.cache = cache self.response_cache = Cache(expiration_in_seconds=expiration_in_seconds) self.request_timeout_in_seconds = request_timeout_in_seconds self.connect_timeout_in_seconds = connect_timeout_in_seconds self.ignore_pycurl = ignore_pycurl self.running_urls = 0 self.url_queue = [] if PYCURL_AVAILABLE and not self.ignore_pycurl: logging.debug('pycurl is available, thus Octopus will be using it instead of tornado\'s simple http client.') AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") self.allow_connection_reuse = allow_connection_reuse else: self.allow_connection_reuse = True if auto_start: logging.debug('Auto starting...') self.start() self.limiter = limiter
def main(): def scrapy(body): bs = BeautifulSoup(body) print(bs) item = bs.find("div",id="product-intro").find("div",id="itemInfo") name = item.find("div",id="name").find("h1").text price = item.find("div",id="summary-price").find("strong",class_="p-price").text #print(price) price = re_price.search(price).group() discount = item.find("div",id="summary-price").find("span",class_="p-discount").text #print(discount) discount = re_price.search(discount).group() #print(name,price,discount) print("insert a shop(%s,%s,%s)"%(name,price,discount)) def prepare_cul_opts(obj): pass #obj.setopt(pycurl.WRITEFUNCTION,open("result.html","wb").write) #obj.setopt(pycurl.NOBODY,True) #url="http://upload-images.jianshu.io/upload_images/1679702-7e810a34f3ef8d18.jpg?imageMogr2/auto-orient/strip%7CimageView2/1/w/300/h/300" AsyncHTTPClient.configure(CurlAsyncHTTPClient) for url in ["http://item.jd.com/11917788.html"]: httpCli = AsyncHTTPClient() respone = yield httpCli.fetch(url,prepare_curl_callback=prepare_cul_opts) scrapy(respone.body) print(list(respone.headers.get_all()))
def __init__(self, url, callback = None, retry = 0, keep_alive = False, ssl = False, validate_cert = False, user = None, password = None): """ Build the event source client :param url: string, the url to connect to :param action: string of the listening action to connect to :param target: string with the listening token :param callback: function with one parameter (Event) that gets called for each received event :param retry: timeout between two reconnections (0 means no reconnection) """ log.debug("EventSourceClient(%s,%s,%s)" % (url, callback, retry)) self.data_partial = None self.last_event_id = None self.retry_timeout = int(retry) self.keep_alive = keep_alive self._url = url self._headers = {"Accept": "text/event-stream"} self._user = user self._password = password AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") self.http_client = AsyncHTTPClient() self.http_request = HTTPRequest(url = self._url, method="GET", headers={"content-type":"text/event-stream"}, request_timeout = 0, validate_cert = validate_cert, streaming_callback = self.handle_stream, auth_username = user, auth_password = password) if callback is None: self.cb = lambda e: log.info( "received %s" % (e,) ) else: self.cb = callback
def test_max_clients(self): # The max_clients argument is tricky because it was originally # allowed to be passed positionally; newer arguments are keyword-only. AsyncHTTPClient.configure(SimpleAsyncHTTPClient) with closing(AsyncHTTPClient( self.io_loop, force_instance=True)) as client: self.assertEqual(client.max_clients, 10) with closing(AsyncHTTPClient( self.io_loop, 11, force_instance=True)) as client: self.assertEqual(client.max_clients, 11) with closing(AsyncHTTPClient( self.io_loop, max_clients=11, force_instance=True)) as client: self.assertEqual(client.max_clients, 11) # Now configure max_clients statically and try overriding it # with each way max_clients can be passed AsyncHTTPClient.configure(SimpleAsyncHTTPClient, max_clients=12) with closing(AsyncHTTPClient( self.io_loop, force_instance=True)) as client: self.assertEqual(client.max_clients, 12) with closing(AsyncHTTPClient( self.io_loop, max_clients=13, force_instance=True)) as client: self.assertEqual(client.max_clients, 13) with closing(AsyncHTTPClient( self.io_loop, max_clients=14, force_instance=True)) as client: self.assertEqual(client.max_clients, 14)
def tsend_log(self, message, severity, filename=None, url=None, status_code=None, headers=None, parameters=None, stacktrace=False): from tornado.httpclient import AsyncHTTPClient d = self._build_message( message, severity, filename, url, status_code, headers, parameters, stacktrace) # want to use the better client here. AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") AsyncHTTPClient().fetch( self._build_url(), lambda resp: None, method="POST", body=json.dumps(d), headers=headers)
def client(io_loop, request): """Return mocked AsyncHTTPClient""" before = AsyncHTTPClient.configured_class() AsyncHTTPClient.configure(MockAsyncHTTPClient) request.addfinalizer(lambda : AsyncHTTPClient.configure(before)) c = AsyncHTTPClient() assert isinstance(c, MockAsyncHTTPClient) return c
def get_http_client(self): """Overrides `AsyncHTTPTestCase.get_http_client` to separate unit test HTTPClient from application HTTPClient. This allows mocking HTTP requests made by application in unit tests. """ AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient') return AsyncHTTPClient(force_instance=True)
def _enable_curl_httpclient(cls): """ Tornado proxies are currently only supported with curl_httpclient http://www.tornadoweb.org/en/stable/httpclient.html#request-objects """ if not cls._curl_httpclient_enabled: AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") cls._curl_httpclient_enabled = True
def run(self): print("TornadoThread Start....") AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") app = tornado.web.Application(tornado_handlers, template_path=template_path, static_path=static_path, **settings) app.listen(4000, address="0.0.0.0") tornado.ioloop.IOLoop.current().start() return
def main(): """A simple test runner. This test runner is essentially equivalent to `unittest.main` from the standard library, but adds support for tornado-style option parsing and log formatting. The easiest way to run a test is via the command line:: python -m tornado.testing tornado.test.stack_context_test See the standard library unittest module for ways in which tests can be specified. Projects with many tests may wish to define a test script like tornado/test/runtests.py. This script should define a method all() which returns a test suite and then call tornado.testing.main(). Note that even when a test script is used, the all() test suite may be overridden by naming a single test on the command line:: # Runs all tests tornado/test/runtests.py # Runs one test tornado/test/runtests.py tornado.test.stack_context_test """ from tornado.options import define, options, parse_command_line define('autoreload', type=bool, default=False, help="DEPRECATED: use tornado.autoreload.main instead") define('httpclient', type=str, default=None) argv = [sys.argv[0]] + parse_command_line(sys.argv) if options.httpclient: from tornado.httpclient import AsyncHTTPClient AsyncHTTPClient.configure(options.httpclient) if __name__ == '__main__' and len(argv) == 1: print >> sys.stderr, "No tests specified" sys.exit(1) try: # In order to be able to run tests by their fully-qualified name # on the command line without importing all tests here, # module must be set to None. Python 3.2's unittest.main ignores # defaultTest if no module is given (it tries to do its own # test discovery, which is incompatible with auto2to3), so don't # set module if we're not asking for a specific test. if len(argv) > 1: unittest.main(module=None, argv=argv) else: unittest.main(defaultTest="all", argv=argv) except SystemExit, e: if e.code == 0: logging.info('PASS') else: logging.error('FAIL') if not options.autoreload: raise
def setup(): """Set up options and logger. Configure the asynchronous HTTP client.""" define( 'apiurl', type=str, help='The Juju WebSocket server address. This is usually the address ' 'of the bootstrap/state node as returned by "juju status".') # Optional parameters. define( 'apiversion', type=str, default=DEFAULT_API_VERSION, help='the Juju API version/implementation. Currently the possible ' 'values are "go" (default) or "python".') define( 'testsroot', type=str, help='The filesystem path of the Juju GUI tests directory. ' 'If not provided, tests are not served.') define( 'sslpath', type=str, default=DEFAULT_SSL_PATH, help='The path where the SSL certificates are stored.') define( 'insecure', type=bool, default=False, help='Set to True to serve the GUI over an insecure HTTP connection. ' 'Do not set unless you understand and accept the risks.') define( 'sandbox', type=bool, default=False, help='Set to True if the GUI is running in sandbox mode, i.e. using ' 'an in-memory backend. When this is set to True, the GUI server ' 'does not listen to incoming WebSocket connections, and ' 'therefore the --apiurl and --apiversion options are ignored.') define( 'charmworldurl', type=str, help='The URL to use for Charmworld.') define( 'port', type=int, help='User defined port to run the server on. If no port is defined ' 'the server will be started on 80 and 443 as per the default ' 'port options from the charm.') define( 'jujuguidebug', type=bool, default=False, help='Set to True to run the gui without minifiying or combining ' 'source files.') define('user', type=str, help='The juju environment user.') define('password', type=str, help='The juju environment password.') define('uuid', type=str, help='The juju environment uuid.') define('jujuversion', type=str, help='The jujud version.') define( 'jemlocation', type=str, help="The url for a Juju Environment Manager.") define( 'interactivelogin', type=bool, default=False, help='Enables interactive login to identity manager, if applicable.') # In Tornado, parsing the options also sets up the default logger. parse_command_line() _validate_choices('apiversion', ('go', 'python')) _validate_range('port', 1, 65535) _add_debug(logging.getLogger()) # Configure the asynchronous HTTP client used by proxy handlers. AsyncHTTPClient.configure( 'tornado.curl_httpclient.CurlAsyncHTTPClient', max_clients=20)
def main(): """A simple test runner with autoreload support. The easiest way to run a test is via the command line:: python -m tornado.testing --autoreload tornado.test.stack_context_test See the standard library unittest module for ways in which tests can be specified. Projects with many tests may wish to define a test script like tornado/test/runtests.py. This script should define a method all() which returns a test suite and then call tornado.testing.main(). Note that even when a test script is used, the all() test suite may be overridden by naming a single test on the command line:: # Runs all tests tornado/test/runtests.py --autoreload # Runs one test tornado/test/runtests.py --autoreload tornado.test.stack_context_test If --autoreload is specified, the process will continue running after the tests finish, and when any source file changes the tests will be rerun. Without --autoreload, the process will exit once the tests finish (with an exit status of 0 for success and non-zero for failures). """ from tornado.options import define, options, parse_command_line define('autoreload', type=bool, default=False) define('httpclient', type=str, default=None) argv = [sys.argv[0]] + parse_command_line(sys.argv) if options.httpclient: from tornado.httpclient import AsyncHTTPClient AsyncHTTPClient.configure(options.httpclient) if __name__ == '__main__' and len(argv) == 1: print >> sys.stderr, "No tests specified" sys.exit(1) try: # In order to be able to run tests by their fully-qualified name # on the command line without importing all tests here, # module must be set to None. Python 3.2's unittest.main ignores # defaultTest if no module is given (it tries to do its own # test discovery, which is incompatible with auto2to3), so don't # set module if we're not asking for a specific test. if len(argv) > 1: unittest.main(module=None, argv=argv) else: unittest.main(defaultTest="all", argv=argv) except SystemExit, e: if e.code == 0: logging.info('PASS') else: logging.error('FAIL') if not options.autoreload: raise
def __init__(self, url, key, sender, **kwargs): AsyncHTTPClient.configure( 'tornado.curl_httpclient.CurlAsyncHTTPClient' ) self.url = url self.key = key self.sender = sender self.kwargs = kwargs self.http_client = AsyncHTTPClient()
def configure_app(self, config=None, log_level='INFO', debug=False, main_loop=None): self.config = config self.main_loop = main_loop handlers = [ url(r'/api/(?P<project_id>\d+)/store/', RouterHandler, name="router"), # Deprecated url(r'/api/store/', OldRouterHandler, name="router_post"), #/Deprecated url(r'/count', CountHandler, name="count"), url(r'/healthcheck(?:/|\.html)?', HealthCheckHandler, name="healthcheck"), ] logging.info("Connecting to db on {0}:{1} on database {2} with user {3}".format( self.config.MYSQL_HOST, self.config.MYSQL_PORT, self.config.MYSQL_DB, self.config.MYSQL_USER) ) cache_class = get_class(self.config.CACHE_IMPLEMENTATION_CLASS) self.cache = cache_class(self) storage_class = get_class(self.config.STORAGE) self.storage = storage_class(self) options = {} self.project_keys = {} self.processed_items = 0 self.ignored_items = 0 #if self.config.PROCESS_NEWER_MESSAGES_FIRST: #self.items_to_process = defaultdict(LifoQueue) #else: #self.items_to_process = defaultdict(Queue) self.last_requests = [] self.average_request_time = None self.percentile_request_time = None projects_update_task = ProjectsUpdateTask(self, self.main_loop) projects_update_task.update() projects_update_task.start() send_to_sentry_task = SendToSentryTask(self, self.main_loop) send_to_sentry_task.update() send_to_sentry_task.start() if debug: options['debug'] = True config.NUMBER_OF_FORKS = 1 AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") return handlers, options
def __init__(self, dispatcher_client, bindaddr='127.0.0.1', port=8080, certfile=None, keyfile=None): self._port = port self._client = dispatcher_client self._bindaddr = bindaddr self._certfile = certfile self._keyfile = keyfile self._ioloop = None self._server = None AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
def __init__(self, endpoint, token=None): self.endpoint = endpoint self.token = token if endpoint.startswith("http"): self._base_url = self.endpoint else: self._base_url = "https://%s" % self.endpoint AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient", defaults=dict(validate_cert=False))
def main(): parser = argparse.ArgumentParser() parser.add_argument("input", help="input: list of urls") parser.add_argument("--pool-size", help="max count of urls that can be resolved simultaneously", type=int, default=10) args = parser.parse_args() AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") io_loop = ioloop.IOLoop.instance() async_client = AsyncHTTPClient() pool = WorkPool(args.input, async_client,io_loop=io_loop, pool_size=args.pool_size) io_loop.run_sync(pool.run)
def config(): with open('config.yaml', 'r') as f: config = yaml.load(f) for k,v in config.items(): logging.info("%s:%s", k,v) define(k, default=v) # curl_httpclient is faster, it is said max_clients = options.max_fetch_clients + options.max_send_clients AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient", max_clients=max_clients)
def __init__(self, *args, **kwargs): super(PartyuApp, self).__init__(*args, **kwargs) self.db = None AsyncHTTPClient.configure(None, max_clients=options.http_max_clients) self.client = AsyncHTTPClient() self.comms = {} self.comms["fsq"] = FoursquareComm(self.client) self.comms["fb"] = FacebookComm(self.client)
def async_purge(self, files=None, max_concurrency=None, isVerbose=False): """Asynchronously purge all the files from the Fastly CDN cache.""" self.files = files self.isVerbose = isVerbose self.http_client = AsyncHTTPClient() AsyncHTTPClient.configure(None, max_clients=max_concurrency) workers = [] for i in range(min(len(self.files), max_concurrency)): workers.append(self.purge_worker()) yield workers
def __init__(self, policy): """ :type policy: ScrapingPolicy """ self.policy = policy self.requests_in_flight = 0 self.crawled = set() self.backlog = set() self.redirects = {} # maps URLs to their destination self.errors = {} # maps URLs to their (error) HTTP status codes AsyncHTTPClient.configure(None, defaults=dict(user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36')) self.http_client = AsyncHTTPClient(max_clients=policy.max_concurrent_requests)
def run(url_to_handlers=default_handlers): define("port", default=9999, help="Runs on the given port", type=int) define("secret", default='', help="Shared secret for /killexecutor", type=str) parse_command_line() logger = logging.getLogger(__file__) logger.info("Starting Heron Shell") logger.info("Shared secret for /killexecutor: %s", options.secret) AsyncHTTPClient.configure(None, defaults=dict(request_timeout=120.0)) app = tornado.web.Application(url_to_handlers) app.listen(options.port) tornado.ioloop.IOLoop.instance().start()
def __init__(self, config): AsyncHTTPClient.configure(None, defaults=dict(request_timeout=120.0)) self.tracker = Tracker(config) self.tracker.synch_topologies() tornadoHandlers = [ (r"/", handlers.MainHandler), (r"/clusters", handlers.ClustersHandler, { "tracker": self.tracker }), (r"/topologies", handlers.TopologiesHandler, { "tracker": self.tracker }), (r"/topologies/states", handlers.StatesHandler, { "tracker": self.tracker }), (r"/topologies/info", handlers.TopologyHandler, { "tracker": self.tracker }), (r"/topologies/logicalplan", handlers.LogicalPlanHandler, { "tracker": self.tracker }), (r"/topologies/containerfiledata", handlers.ContainerFileDataHandler, { "tracker": self.tracker }), (r"/topologies/containerfiledownload", handlers.ContainerFileDownloadHandler, { "tracker": self.tracker }), (r"/topologies/containerfilestats", handlers.ContainerFileStatsHandler, { "tracker": self.tracker }), (r"/topologies/physicalplan", handlers.PhysicalPlanHandler, { "tracker": self.tracker }), # Deprecated. See https://github.com/twitter/heron/issues/1754 (r"/topologies/executionstate", handlers.ExecutionStateHandler, { "tracker": self.tracker }), (r"/topologies/schedulerlocation", handlers.SchedulerLocationHandler, { "tracker": self.tracker }), (r"/topologies/metadata", handlers.MetaDataHandler, { "tracker": self.tracker }), (r"/topologies/runtimestate", handlers.RuntimeStateHandler, { "tracker": self.tracker }), (r"/topologies/metrics", handlers.MetricsHandler, { "tracker": self.tracker }), (r"/topologies/metricstimeline", handlers.MetricsTimelineHandler, { "tracker": self.tracker }), (r"/topologies/metricsquery", handlers.MetricsQueryHandler, { "tracker": self.tracker }), (r"/topologies/exceptions", handlers.ExceptionHandler, { "tracker": self.tracker }), (r"/topologies/exceptionsummary", handlers.ExceptionSummaryHandler, { "tracker": self.tracker }), (r"/machines", handlers.MachinesHandler, { "tracker": self.tracker }), (r"/topologies/pid", handlers.PidHandler, { "tracker": self.tracker }), (r"/topologies/jstack", handlers.JstackHandler, { "tracker": self.tracker }), (r"/topologies/jmap", handlers.JmapHandler, { "tracker": self.tracker }), (r"/topologies/histo", handlers.MemoryHistogramHandler, { "tracker": self.tracker }), (r"(.*)", handlers.DefaultHandler), ] settings = dict(debug=True, serve_traceback=True, static_path=os.path.dirname(__file__)) tornado.web.Application.__init__(self, tornadoHandlers, **settings) Log.info("Tracker has started")
def configure_httpclient(): if options.httpclient is not None: AsyncHTTPClient.configure(options.httpclient) else: AsyncHTTPClient.configure('tornado_http2.client.ForceHTTP2Client')
import json import logging from collections import defaultdict from tornado.gen import coroutine from tornado.gen import Return from tornado.httpclient import AsyncHTTPClient from api import conf as consts from api.extension.client import NoQueueTimeoutHTTPClient LOG = logging.getLogger(__name__) AsyncHTTPClient.configure(NoQueueTimeoutHTTPClient) class Result(object): def __init__(self): self._url = '{}/results?period={}&version={}&page={}' self._client = AsyncHTTPClient() self._result = defaultdict(list) @property @coroutine def result(self): if not self._result: yield self.update_results() raise Return(self._result) @coroutine def update_results(self):
reason) = httputil.parse_response_start_line(header_line) header_line = "X-Http-Reason: %s" % reason except httputil.HTTPInputError: return if not header_line: return headers.parse_line(header_line) def _curl_debug(self, debug_type: int, debug_msg: str) -> None: debug_types = ("I", "<", ">", "<", ">") if debug_type == 0: debug_msg = native_str(debug_msg) curl_log.debug("%s", debug_msg.strip()) elif debug_type in (1, 2): debug_msg = native_str(debug_msg) for line in debug_msg.splitlines(): curl_log.debug("%s %s", debug_types[debug_type], line) elif debug_type == 4: curl_log.debug("%s %r", debug_types[debug_type], debug_msg) class CurlError(HTTPError): def __init__(self, errno: int, message: str) -> None: HTTPError.__init__(self, 599, message) self.errno = errno if __name__ == "__main__": AsyncHTTPClient.configure(CurlAsyncHTTPClient) main()
def main(): # Be strict about most warnings (This is set in our test running # scripts to catch import-time warnings, but set it again here to # be sure). This also turns on warnings that are ignored by # default, including DeprecationWarnings and python 3.2's # ResourceWarnings. warnings.filterwarnings("error") # setuptools sometimes gives ImportWarnings about things that are on # sys.path even if they're not being used. warnings.filterwarnings("ignore", category=ImportWarning) # Tornado generally shouldn't use anything deprecated, but some of # our dependencies do (last match wins). warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("error", category=DeprecationWarning, module=r"tornado\..*") warnings.filterwarnings("ignore", category=PendingDeprecationWarning) warnings.filterwarnings("error", category=PendingDeprecationWarning, module=r"tornado\..*") # The unittest module is aggressive about deprecating redundant methods, # leaving some without non-deprecated spellings that work on both # 2.7 and 3.2 warnings.filterwarnings("ignore", category=DeprecationWarning, message="Please use assert.* instead") warnings.filterwarnings( "ignore", category=PendingDeprecationWarning, message="Please use assert.* instead", ) # Twisted 15.0.0 triggers some warnings on py3 with -bb. warnings.filterwarnings("ignore", category=BytesWarning, module=r"twisted\..*") if (3, ) < sys.version_info < (3, 6): # Prior to 3.6, async ResourceWarnings were rather noisy # and even # `python3.4 -W error -c 'import asyncio; asyncio.get_event_loop()'` # would generate a warning. warnings.filterwarnings("ignore", category=ResourceWarning, module=r"asyncio\..*") # This deprecation warning is introduced in Python 3.8 and is # triggered by pycurl. Unforunately, because it is raised in the C # layer it can't be filtered by module and we must match the # message text instead (Tornado's C module uses PY_SSIZE_T_CLEAN # so it's not at risk of running into this issue). warnings.filterwarnings( "ignore", category=DeprecationWarning, message="PY_SSIZE_T_CLEAN will be required", ) logging.getLogger("tornado.access").setLevel(logging.CRITICAL) define( "httpclient", type=str, default=None, callback=lambda s: AsyncHTTPClient.configure( s, defaults=dict(allow_ipv6=False)), ) define("httpserver", type=str, default=None, callback=HTTPServer.configure) define("resolver", type=str, default=None, callback=Resolver.configure) define( "debug_gc", type=str, multiple=True, help="A comma-separated list of gc module debug constants, " "e.g. DEBUG_STATS or DEBUG_COLLECTABLE,DEBUG_OBJECTS", callback=lambda values: gc.set_debug( reduce(operator.or_, (getattr(gc, v) for v in values))), ) def set_locale(x): locale.setlocale(locale.LC_ALL, x) define("locale", type=str, default=None, callback=set_locale) log_counter = LogCounter() add_parse_callback( lambda: logging.getLogger().handlers[0].addFilter(log_counter)) # Certain errors (especially "unclosed resource" errors raised in # destructors) go directly to stderr instead of logging. Count # anything written by anything but the test runner as an error. orig_stderr = sys.stderr counting_stderr = CountingStderr(orig_stderr) sys.stderr = counting_stderr # type: ignore import tornado.testing kwargs = {} # HACK: unittest.main will make its own changes to the warning # configuration, which may conflict with the settings above # or command-line flags like -bb. Passing warnings=False # suppresses this behavior, although this looks like an implementation # detail. http://bugs.python.org/issue15626 kwargs["warnings"] = False kwargs["testRunner"] = test_runner_factory(orig_stderr) try: tornado.testing.main(**kwargs) finally: # The tests should run clean; consider it a failure if they # logged anything at info level or above. if (log_counter.info_count > 0 or log_counter.warning_count > 0 or log_counter.error_count > 0 or counting_stderr.byte_count > 0): logging.error( "logged %d infos, %d warnings, %d errors, and %d bytes to stderr", log_counter.info_count, log_counter.warning_count, log_counter.error_count, counting_stderr.byte_count, ) sys.exit(1)
def __init__(self, base_url): # pylint: disable=fixme # TODO: hacky solution # sys.path[0] should be the path to the extracted files for heron-ui, as it is added # when bootstrapping the pex file static_prefix = '/static/' if base_url != "": static_prefix = os.path.join(base_url, 'static/') AsyncHTTPClient.configure(None, defaults=dict(request_timeout=120.0)) Log.info("Using base url: %s", base_url) settings = dict( template_path=os.path.join(sys.path[0], "heron/tools/ui/resources/templates"), static_path=os.path.join(sys.path[0], "heron/tools/ui/resources/static"), static_url_prefix=static_prefix, gzip=True, debug=True, default_handler_class=handlers.NotFoundHandler, ) Log.info(os.path.join(base_url, 'static/')) # Change these to query string parameters, since # current format can lead to pattern matching issues. callbacks = [ (r"/", handlers.MainHandler), url(r"/topologies", handlers.ListTopologiesHandler, dict(baseUrl=base_url), name='topologies'), url(r"/topologies/filestats/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)", handlers.ContainerFileStatsHandler, dict(baseUrl=base_url)), url(r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/config", handlers.TopologyConfigHandler, dict(baseUrl=base_url)), url( r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/exceptions", handlers.TopologyExceptionsPageHandler, dict(baseUrl=base_url)), url(r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)", handlers.TopologyPlanHandler, dict(baseUrl=base_url)), # topology metric apis (r"/topologies/metrics", handlers.api.MetricsHandler), (r"/topologies/metrics/timeline", handlers.api.MetricsTimelineHandler), url(r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/file", handlers.ContainerFileHandler, dict(baseUrl=base_url)), url(r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/filedata", handlers.ContainerFileDataHandler, dict(baseUrl=base_url)), url( r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/filedownload", handlers.ContainerFileDownloadHandler, dict(baseUrl=base_url)), # Topology list and plan handlers (r"/topologies/list.json", handlers.api.ListTopologiesJsonHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/logicalplan.json", handlers.api.TopologyLogicalPlanJsonHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/physicalplan.json", handlers.api.TopologyPhysicalPlanJsonHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/executionstate.json", handlers.api.TopologyExecutionStateJsonHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/schedulerlocation.json", handlers.api.TopologySchedulerLocationJsonHandler), # Counter Handlers (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/exceptions.json", handlers.api.TopologyExceptionsJsonHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/exceptionsummary.json", handlers.api.TopologyExceptionSummaryHandler), # Heron shell Handlers (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/pid", handlers.api.PidHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/jstack", handlers.api.JstackHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/jmap", handlers.api.JmapHandler), (r"/topologies/([^\/]+)/([^\/]+)/([^\/]+)/([^\/]+)/histo", handlers.api.MemoryHistogramHandler), ## Static files (r"/static/(.*)", tornado.web.StaticFileHandler, dict(path=settings['static_path'])) ] tornado.web.Application.__init__(self, callbacks, **settings)
from tornado import ioloop from tornado.httpclient import AsyncHTTPClient from tornado import gen from functools import partial import string import random AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient", max_clients=100) def generate_urls(base_url, num_urls): for i in range(num_urls): yield base_url + "".join(random.sample(string.ascii_lowercase, 10)) @gen.coroutine def run_experiment(base_url, num_iter=500): http_client = AsyncHTTPClient() urls = generate_urls(base_url, num_iter) responses = yield [http_client.fetch(url) for url in urls] response_sum = sum(len(r.body) for r in responses) raise gen.Return(value=response_sum) if __name__ == "__main__": import time delay = 100 num_iter = 500 _ioloop = ioloop.IOLoop.instance() run_func = partial(
def _default_client(self): ssl_context = make_ssl_context(self.keyfile, self.certfile, cafile=self.client_ca) AsyncHTTPClient.configure(None, defaults={"ssl_options": ssl_context}) return AsyncHTTPClient()
def main(): define( 'url', default=os.environ.get('JUPYTERHUB_API_URL'), help="The JupyterHub API URL", ) define('timeout', default=600, help="The idle timeout (in seconds)") define( 'cull_every', default=0, help="The interval (in seconds) for checking for idle servers to cull", ) define( 'max_age', default=0, help= "The maximum age (in seconds) of servers that should be culled even if they are active", ) define( 'cull_users', default=False, help="""Cull users in addition to servers. This is for use in temporary-user cases such as tmpnb.""", ) define( 'concurrency', default=10, help="""Limit the number of concurrent requests made to the Hub. Deleting a lot of users at the same time can slow down the Hub, so limit the number of API requests we have outstanding at any given time. """, ) define( 'hooks_dir', default="/srv/jupyterhub/culler", help= "Path to the directory for the krb tickets scripts (check_ticket.sh and delete_ticket.sh)" ) define( 'disable_hooks', default=False, help= "The user's home is a temporary scratch directory and we should not check krb tickets" ) parse_command_line() if not options.cull_every: options.cull_every = options.timeout // 2 api_token = os.environ['JUPYTERHUB_API_TOKEN'] try: AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient") except ImportError as e: app_log.warning( "Could not load pycurl: %s\n" "pycurl is recommended if you have a large number of users.", e, ) loop = IOLoop.current() cull = partial( cull_idle, url=options.url, api_token=api_token, inactive_limit=options.timeout, cull_users=options.cull_users, disable_hooks=options.disable_hooks, max_age=options.max_age, concurrency=options.concurrency, ) # schedule first cull immediately # because PeriodicCallback doesn't start until the end of the first interval loop.add_callback(cull) # schedule periodic cull pc = PeriodicCallback(cull, 1e3 * options.cull_every) pc.start() try: loop.start() except KeyboardInterrupt: pass
def main(): # The -W command-line option does not work in a virtualenv with # python 3 (as of virtualenv 1.7), so configure warnings # programmatically instead. import warnings # Be strict about most warnings. This also turns on warnings that are # ignored by default, including DeprecationWarnings and # python 3.2's ResourceWarnings. warnings.filterwarnings("error") # setuptools sometimes gives ImportWarnings about things that are on # sys.path even if they're not being used. warnings.filterwarnings("ignore", category=ImportWarning) # Tornado generally shouldn't use anything deprecated, but some of # our dependencies do (last match wins). warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("error", category=DeprecationWarning, module=r"tornado\..*") warnings.filterwarnings("ignore", category=PendingDeprecationWarning) warnings.filterwarnings("error", category=PendingDeprecationWarning, module=r"tornado\..*") # The unittest module is aggressive about deprecating redundant methods, # leaving some without non-deprecated spellings that work on both # 2.7 and 3.2 warnings.filterwarnings("ignore", category=DeprecationWarning, message="Please use assert.* instead") # unittest2 0.6 on py26 reports these as PendingDeprecationWarnings # instead of DeprecationWarnings. warnings.filterwarnings("ignore", category=PendingDeprecationWarning, message="Please use assert.* instead") # Twisted 15.0.0 triggers some warnings on py3 with -bb. warnings.filterwarnings("ignore", category=BytesWarning, module=r"twisted\..*") # The __aiter__ protocol changed in python 3.5.2. # Silence the warning until we can drop 3.5.[01]. warnings.filterwarnings("ignore", category=PendingDeprecationWarning, message=".*legacy __aiter__ protocol") # 3.5.2's PendingDeprecationWarning became a DeprecationWarning in 3.6. warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*legacy __aiter__ protocol") logging.getLogger("tornado.access").setLevel(logging.CRITICAL) define('httpclient', type=str, default=None, callback=lambda s: AsyncHTTPClient.configure( s, defaults=dict(allow_ipv6=False))) define('httpserver', type=str, default=None, callback=HTTPServer.configure) define('ioloop', type=str, default=None) define('ioloop_time_monotonic', default=False) define('resolver', type=str, default=None, callback=Resolver.configure) define('debug_gc', type=str, multiple=True, help="A comma-separated list of gc module debug constants, " "e.g. DEBUG_STATS or DEBUG_COLLECTABLE,DEBUG_OBJECTS", callback=lambda values: gc.set_debug( reduce(operator.or_, (getattr(gc, v) for v in values)))) define('locale', type=str, default=None, callback=lambda x: locale.setlocale(locale.LC_ALL, x)) def configure_ioloop(): kwargs = {} if options.ioloop_time_monotonic: from tornado.platform.auto import monotonic_time if monotonic_time is None: raise RuntimeError("monotonic clock not found") kwargs['time_func'] = monotonic_time if options.ioloop or kwargs: IOLoop.configure(options.ioloop, **kwargs) add_parse_callback(configure_ioloop) log_counter = LogCounter() add_parse_callback( lambda: logging.getLogger().handlers[0].addFilter(log_counter)) import tornado.testing kwargs = {} if sys.version_info >= (3, 2): # HACK: unittest.main will make its own changes to the warning # configuration, which may conflict with the settings above # or command-line flags like -bb. Passing warnings=False # suppresses this behavior, although this looks like an implementation # detail. http://bugs.python.org/issue15626 kwargs['warnings'] = False kwargs['testRunner'] = TornadoTextTestRunner try: tornado.testing.main(**kwargs) finally: # The tests should run clean; consider it a failure if they logged # any warnings or errors. We'd like to ban info logs too, but # we can't count them cleanly due to interactions with LogTrapTestCase. if log_counter.warning_count > 0 or log_counter.error_count > 0: logging.error("logged %d warnings and %d errors", log_counter.warning_count, log_counter.error_count) sys.exit(1)
buffer = BytesIO(data) # TODO: don't require one big string? response = HTTPResponse( original_request, self.code, reason=getattr(self, "reason", None), headers=self.headers, request_time=self.io_loop.time() - self.start_time, start_time=self.start_wall_time, buffer=buffer, effective_url=self.request.url, ) self._run_callback(response) self._on_end_request() def _on_end_request(self) -> None: self.stream.close() def data_received(self, chunk: bytes) -> None: if self._should_follow_redirect(): # We're going to follow a redirect so just discard the body. return if self.request.streaming_callback is not None: self.request.streaming_callback(chunk) else: self.chunks.append(chunk) if __name__ == "__main__": AsyncHTTPClient.configure(SimpleAsyncHTTPClient) main()
def main(): # command-line options define("debug", default=False, help="run in debug mode", type=bool) define("no_cache", default=False, help="Do not cache results", type=bool) define( "localfiles", default="", help= "Allow to serve local files under /localfile/* this can be a security risk", type=str) define("port", default=5000, help="run on the given port", type=int) define("cache_expiry_min", default=10 * 60, help="minimum cache expiry (seconds)", type=int) define("cache_expiry_max", default=2 * 60 * 60, help="maximum cache expiry (seconds)", type=int) define("mc_threads", default=1, help="number of threads to use for Async Memcache", type=int) define("threads", default=1, help="number of threads to use for rendering", type=int) define("processes", default=0, help="use processes instead of threads for rendering", type=int) define("frontpage", default=FRONTPAGE_JSON, help="path to json file containing frontpage content", type=str) define("sslcert", help="path to ssl .crt file", type=str) define("sslkey", help="path to ssl .key file", type=str) define("default_format", default="html", help="format to use for legacy / URLs", type=str) define("proxy_host", default="", help="The proxy URL.", type=str) define("proxy_port", default="", help="The proxy port.", type=int) define("providers", default=default_providers, help="Full dotted package(s) that provide `default_handlers`", type=str, multiple=True, group="provider") define("provider_rewrites", default=default_rewrites, help="Full dotted package(s) that provide `uri_rewrites`", type=str, multiple=True, group="provider") define("mathjax_url", default="https://cdn.mathjax.org/mathjax/latest/", help="URL base for mathjax package", type=str) tornado.options.parse_command_line() # NBConvert config config = Config() config.NbconvertApp.fileext = 'html' config.CSSHTMLHeaderTransformer.enabled = False # don't strip the files prefix - we use it for redirects # config.Exporter.filters = {'strip_files_prefix': lambda s: s} # DEBUG env implies both autoreload and log-level if os.environ.get("DEBUG"): options.debug = True logging.getLogger().setLevel(logging.DEBUG) # setup memcache mc_pool = ThreadPoolExecutor(options.mc_threads) # setup formats formats = configure_formats(options, config, log.app_log) if options.processes: pool = ProcessPoolExecutor(options.processes) else: pool = ThreadPoolExecutor(options.threads) memcache_urls = os.environ.get('MEMCACHIER_SERVERS', os.environ.get('MEMCACHE_SERVERS')) # Handle linked Docker containers if (os.environ.get('NBCACHE_PORT')): tcp_memcache = os.environ.get('NBCACHE_PORT') memcache_urls = tcp_memcache.split('tcp://')[1] if (os.environ.get('NBINDEX_PORT')): log.app_log.info("Indexing notebooks") tcp_index = os.environ.get('NBINDEX_PORT') index_url = tcp_index.split('tcp://')[1] index_host, index_port = index_url.split(":") indexer = ElasticSearch(index_host, index_port) else: log.app_log.info("Not indexing notebooks") indexer = NoSearch() if options.no_cache: log.app_log.info("Not using cache") cache = MockCache() elif pylibmc and memcache_urls: kwargs = dict(pool=mc_pool) username = os.environ.get('MEMCACHIER_USERNAME', '') password = os.environ.get('MEMCACHIER_PASSWORD', '') if username and password: kwargs['binary'] = True kwargs['username'] = username kwargs['password'] = password log.app_log.info("Using SASL memcache") else: log.app_log.info("Using plain memecache") cache = AsyncMultipartMemcache(memcache_urls.split(','), **kwargs) else: log.app_log.info("Using in-memory cache") cache = DummyAsyncCache() # setup tornado handlers and settings template_path = pjoin(here, 'templates') static_path = pjoin(here, 'static') env = Environment(loader=FileSystemLoader(template_path), autoescape=True) env.filters['markdown'] = markdown.markdown try: git_data = git_info(here) except Exception as e: app_log.error("Failed to get git info: %s", e) git_data = {} else: git_data['msg'] = escape(git_data['msg']) if options.no_cache: # force jinja to recompile template every time env.globals.update(cache_size=0) env.globals.update( nrhead=nrhead, nrfoot=nrfoot, git_data=git_data, ipython_info=ipython_info(), len=len, ) AsyncHTTPClient.configure(HTTPClientClass) client = AsyncHTTPClient() # load frontpage sections with io.open(options.frontpage, 'r') as f: frontpage_sections = json.load(f) # cache frontpage links for the maximum allowed time max_cache_uris = {''} for section in frontpage_sections: for link in section['links']: max_cache_uris.add('/' + link['target']) fetch_kwargs = dict(connect_timeout=10, ) if options.proxy_host: fetch_kwargs.update( dict(proxy_host=options.proxy_host, proxy_port=options.proxy_port)) log.app_log.info("Using web proxy {proxy_host}:{proxy_port}." "".format(**fetch_kwargs)) settings = dict( log_function=log_request, jinja2_env=env, static_path=static_path, client=client, formats=formats, default_format=options.default_format, providers=options.providers, provider_rewrites=options.provider_rewrites, config=config, index=indexer, cache=cache, cache_expiry_min=options.cache_expiry_min, cache_expiry_max=options.cache_expiry_max, max_cache_uris=max_cache_uris, frontpage_sections=frontpage_sections, pool=pool, gzip=True, render_timeout=20, localfile_path=os.path.abspath(options.localfiles), fetch_kwargs=fetch_kwargs, mathjax_url=options.mathjax_url, ) # handle handlers handlers = init_handlers(formats, options.providers) if options.localfiles: log.app_log.warning( "Serving local notebooks in %s, this can be a security risk", options.localfiles) # use absolute or relative paths: local_handlers = [(r'/localfile/(.*)', LocalFileHandler)] handlers = (local_handlers + format_handlers(formats, local_handlers) + handlers) # load ssl options ssl_options = None if options.sslcert: ssl_options = { 'certfile': options.sslcert, 'keyfile': options.sslkey, } # create and start the app app = web.Application(handlers, debug=options.debug, **settings) http_server = httpserver.HTTPServer(app, xheaders=True, ssl_options=ssl_options) log.app_log.info("Listening on port %i", options.port) http_server.listen(options.port) ioloop.IOLoop.instance().start()
logging.init_log(StreamHandler(logging.INFO, log_format, logging.DATE_FMT_SIMPLE), disable_existing_loggers=not options.display_url) logger = logging.getLogger(__name__) ASYNC_HTTP_CONNECT_TIMEOUT = 60 ASYNC_HTTP_REQUEST_TIMEOUT = 120 ASYNC_HTTP_CLIENT_MAX_CLIENTS = 100 if platform.system() != 'windows': try: # curl_httpclient is faster than simple_httpclient AsyncHTTPClient.configure( 'tornado.curl_httpclient.CurlAsyncHTTPClient', max_clients=ASYNC_HTTP_CLIENT_MAX_CLIENTS) except ImportError: AsyncHTTPClient.configure('tornado.simple_httpclient.AsyncHTTPClient') class ProxyHandler(RequestHandler): @gen.coroutine def get(self): yield self._do_fetch('GET') @gen.coroutine def post(self): yield self._do_fetch('POST') @gen.coroutine
def make_app(): # NBConvert config config = Config() config.NbconvertApp.fileext = 'html' config.CSSHTMLHeaderTransformer.enabled = False # don't strip the files prefix - we use it for redirects # config.Exporter.filters = {'strip_files_prefix': lambda s: s} # DEBUG env implies both autoreload and log-level if os.environ.get("DEBUG"): options.debug = True logging.getLogger().setLevel(logging.DEBUG) # setup memcache mc_pool = ThreadPoolExecutor(options.mc_threads) # setup formats formats = configure_formats(options, config, log.app_log) if options.processes: pool = ProcessPoolExecutor(options.processes) else: pool = ThreadPoolExecutor(options.threads) memcache_urls = os.environ.get('MEMCACHIER_SERVERS', os.environ.get('MEMCACHE_SERVERS') ) # Handle linked Docker containers if(os.environ.get('NBCACHE_PORT')): tcp_memcache = os.environ.get('NBCACHE_PORT') memcache_urls = tcp_memcache.split('tcp://')[1] if(os.environ.get('NBINDEX_PORT')): log.app_log.info("Indexing notebooks") tcp_index = os.environ.get('NBINDEX_PORT') index_url = tcp_index.split('tcp://')[1] index_host, index_port = index_url.split(":") indexer = ElasticSearch(index_host, index_port) else: log.app_log.info("Not indexing notebooks") indexer = NoSearch() if options.no_cache: log.app_log.info("Not using cache") cache = MockCache() elif pylibmc and memcache_urls: kwargs = dict(pool=mc_pool) username = os.environ.get('MEMCACHIER_USERNAME', '') password = os.environ.get('MEMCACHIER_PASSWORD', '') if username and password: kwargs['binary'] = True kwargs['username'] = username kwargs['password'] = password log.app_log.info("Using SASL memcache") else: log.app_log.info("Using plain memecache") cache = AsyncMultipartMemcache(memcache_urls.split(','), **kwargs) else: log.app_log.info("Using in-memory cache") cache = DummyAsyncCache() # setup tornado handlers and settings template_paths = pjoin(here, 'templates') if options.template_path is not None: log.app_log.info("Using custom template path {}".format( options.template_path) ) template_paths = [options.template_path, template_paths] static_path = pjoin(here, 'static') env = Environment( loader=FileSystemLoader(template_paths), autoescape=True ) env.filters['markdown'] = markdown.markdown try: git_data = git_info(here) except Exception as e: app_log.error("Failed to get git info: %s", e) git_data = {} else: git_data['msg'] = escape(git_data['msg']) if options.no_cache: # force jinja to recompile template every time env.globals.update(cache_size=0) env.globals.update(nrhead=nrhead, nrfoot=nrfoot, git_data=git_data, jupyter_info=jupyter_info(), len=len, ) AsyncHTTPClient.configure(HTTPClientClass) client = AsyncHTTPClient() client.cache = cache # load frontpage sections with io.open(options.frontpage, 'r') as f: frontpage_setup = json.load(f) # check if the json has a 'sections' field, otherwise assume it is # just a list of sessions, and provide the defaults for the other # fields if 'sections' not in frontpage_setup: frontpage_setup = {'title': 'nbviewer', 'subtitle': 'A simple way to share Jupyter Notebooks', 'show_input': True, 'sections': frontpage_setup} # cache frontpage links for the maximum allowed time max_cache_uris = {''} for section in frontpage_setup['sections']: for link in section['links']: max_cache_uris.add('/' + link['target']) fetch_kwargs = dict(connect_timeout=10,) if options.proxy_host: fetch_kwargs.update(dict(proxy_host=options.proxy_host, proxy_port=options.proxy_port)) log.app_log.info("Using web proxy {proxy_host}:{proxy_port}." "".format(**fetch_kwargs)) if options.no_check_certificate: fetch_kwargs.update(dict(validate_cert=False)) log.app_log.info("Not validating SSL certificates") # prefer the jhub defined service prefix over the CLI base_url = os.getenv('JUPYTERHUB_SERVICE_PREFIX', options.base_url) rate_limiter = RateLimiter( limit=options.rate_limit, interval=options.rate_limit_interval, cache=cache, ) settings = dict( log_function=log_request, jinja2_env=env, static_path=static_path, static_url_prefix=url_path_join(base_url, '/static/'), client=client, formats=formats, default_format=options.default_format, providers=options.providers, provider_rewrites=options.provider_rewrites, config=config, index=indexer, cache=cache, cache_expiry_min=options.cache_expiry_min, cache_expiry_max=options.cache_expiry_max, max_cache_uris=max_cache_uris, frontpage_setup=frontpage_setup, pool=pool, gzip=True, render_timeout=options.render_timeout, localfile_path=os.path.abspath(options.localfiles), localfile_follow_symlinks=options.localfile_follow_symlinks, localfile_any_user=options.localfile_any_user, fetch_kwargs=fetch_kwargs, mathjax_url=options.mathjax_url, rate_limiter=rate_limiter, statsd_host=options.statsd_host, statsd_port=options.statsd_port, statsd_prefix=options.statsd_prefix, base_url=base_url, google_analytics_id=os.getenv('GOOGLE_ANALYTICS_ID'), hub_api_token=os.getenv('JUPYTERHUB_API_TOKEN'), hub_api_url=os.getenv('JUPYTERHUB_API_URL'), hub_base_url=os.getenv('JUPYTERHUB_BASE_URL'), ipywidgets_base_url=options.ipywidgets_base_url, jupyter_widgets_html_manager_version=options.jupyter_widgets_html_manager_version, jupyter_js_widgets_version=options.jupyter_js_widgets_version, content_security_policy=options.content_security_policy, binder_base_url=options.binder_base_url, ) if options.localfiles: log.app_log.warning("Serving local notebooks in %s, this can be a security risk", options.localfiles) # handle handlers handlers = init_handlers(formats, options.providers, base_url, options.localfiles) # create the app return web.Application(handlers, debug=options.debug, **settings)
def __init__(self, handlers, root_dir=None, static_path=None, template_path=None, create_http_client=True, **settings): self.log_msg_tmpl = '%s %s' # TODO add valiate paths if root_dir is not None: app_static_dir = os.path.join(root_dir, "static") \ if static_path is None else static_path app_template_dir = os.path.join(root_dir, "templates") \ if template_path is None else template_path else: app_template_dir = app_static_dir = None super().__init__(handlers, static_path=app_static_dir, template_path=app_template_dir, **settings) self.server_name = options.srv_name self.logger = tornado.log.gen_log self.redis_connection_pool = None tornado.ioloop.IOLoop.configure( 'tornado.platform.asyncio.AsyncIOMainLoop') if options.use_mail_logging: if options.log_mail_user == '' and options.log_mail_psw == '': credentials_list = None else: credentials_list = [ options.log_mail_user, options.log_mail_psw ] self.set_mail_logging(options.log_mail_host, options.log_mail_from, options.log_mail_to, options.log_mail_subj, credentials_list) if options.use_reactjs: if not jinja2_import: self.react_env = self.react_assets = None raise ImportError('Required package jinja2 is missing') else: self.react_env = Environment( loader=FileSystemLoader('templates')) self.react_assets = self.load_react_assets() else: self.react_env = self.react_assets = None if options.use_curl_http_client: self.log_debug(options.use_curl_http_client, grep_label='use_curl_http_client') if pycurl is None: raise ImportError('Required package for pycurl ' 'CurlAsyncHTTPClient is missing') else: self.log_debug('configure curl') AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient") self.http_client = AsyncHTTPClient(max_clients=options.max_http_clients ) if create_http_client else None if options.use_mongo: try: self.mongo_pool = get_mongo_pool( options.mongo_db_name, options.mongo_user, options.mongo_psw, options.mongo_auth_db_name, options.mongo_server, options.mongo_port, options.mongo_min_pool_size, options.mongo_max_pool_size) except ModuleNotFoundError: self.mongo_pool = None raise else: self.mongo_pool = None self.event_writer = TorskelEventLogController()
def configure_tornado(): from tornado.httpclient import AsyncHTTPClient AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient', max_clients=16)
if __name__ == '__main__': define('db', type=str, help='DB connection DSN', default=environ.get( 'DB', "dbname=boterator user=boterator host=localhost port=5432")) define('burlesque', type=str, help='Burlesque address', default=environ.get('BURLESQUE', 'http://127.0.0.1:4401')) define('debug', type=bool, default=False) parse_command_line() AsyncHTTPClient.configure(None, max_clients=1024) ioloop = IOLoop.instance() db = Pool(dsn=options.db, size=1, max_size=10, auto_shrink=True, ioloop=IOLoop.current()) ioloop.run_sync(db.connect) if options.debug: autoreload.start() sh = SlaveHolder(db, Burlesque(options.burlesque)) try:
def initialize(self, notebook_store, security_token): self.notebook_store = notebook_store self.security_token = security_token AsyncHTTPClient.configure(None, max_body_size=4000000000)
import os import glob from tornado.httpclient import AsyncHTTPClient from kubernetes import client from z2jh import get_config, get_secret # Configure JupyterHub to use the curl backend for making HTTP requests, # rather than the pure-python implementations. The default one starts # being too slow to make a large number of requests to the proxy API # at the rate required. AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") c.JupyterHub.spawner_class = 'kubespawner.KubeSpawner' # Connect to a proxy running in a different pod c.ConfigurableHTTPProxy.api_url = 'http://{}:{}'.format( os.environ['PROXY_API_SERVICE_HOST'], int(os.environ['PROXY_API_SERVICE_PORT'])) c.ConfigurableHTTPProxy.should_start = False # Do not shut down user pods when hub is restarted c.JupyterHub.cleanup_servers = False # Check that the proxy has routes appropriately setup # This isn't the best named setting :D c.JupyterHub.last_activity_interval = 60 # Max number of servers that can be spawning at any one time c.JupyterHub.concurrent_spawn_limit = get_config('hub.concurrent-spawn-limit')