def __init__(self, token, api_url, http_client=None): super(ProxyTarget, self).__init__(token) self.api_url = api_url self.http_client = http_client or AsyncHTTPClient()
def zufang_spider(url): response = yield AsyncHTTPClient().fetch(url, raise_error=False) raise gen.Return(response)
def get(self, kind, path): # builder vs. viewer enable_editing = (kind == 'builder') # Builder options builder_kwargs = {} for a in [ 'starting_reaction', 'model_name', 'map_name', 'map_json', 'reaction_no_data_color', 'reaction_no_data_size', 'metabolite_no_data_color', 'metabolite_no_data_size', 'hide_secondary_nodes' ]: args = self.get_arguments(a) if len(args) == 1: builder_kwargs[a] = (True if args[0].lower() == 'true' else ( False if args[0].lower() == 'false' else args[0])) # array args for a in [ 'quick_jump', 'metabolite_size_range', 'metabolite_color_range', 'reaction_size_range', 'reaction_color_range', 'gene_styles' ]: args = self.get_arguments(a + '[]') if len(args) > 0: builder_kwargs[a] = args # js source args = self.get_arguments('js_source') js_source = args[0] if len(args) == 1 else 'web' # if the server is running locally, then the embedded css must be loaded # asynchronously using the same server thread. if js_source in ['dev', 'local']: global PORT url = get_url('builder_embed_css', source='local', local_host='http://localhost:%d' % PORT) response = yield gen.Task(AsyncHTTPClient().fetch, url) if response.code != 200 or response.body is None: raise Exception('Could not load embedded_css from %s' % url) builder_kwargs['embedded_css'] = ( response.body.decode('utf-8').replace('\n', ' ')) # example data def load_data_file(rel_path): """Load a JSON file with relative path.""" try: with open(join(root_directory, rel_path), 'r') as f: return json.load(f) except: logging.warn('Could not load example_data file: %s' % rel_path) if len(self.get_arguments('example_data')) > 0: r_filepath = 'escher/example_data/reaction_data_iJO1366.json' builder_kwargs['reaction_data'] = load_data_file(r_filepath) m_filepath = 'escher/example_data/metabolite_data_iJO1366.json' builder_kwargs['metabolite_data'] = load_data_file(m_filepath) # make the builder builder = Builder(safe=True, **builder_kwargs) # display options display_kwargs = { 'minified_js': True, 'scroll_behavior': 'pan', 'menu': 'all' } # keyword for a in [ 'menu', 'scroll_behavior', 'minified_js', 'auto_set_data_domain', 'never_ask_before_quit' ]: args = self.get_arguments(a) if len(args) == 1: display_kwargs[a] = (True if args[0].lower() == 'true' else ( False if args[0].lower() == 'false' else args[0])) html = builder._get_html(js_source=js_source, enable_editing=enable_editing, enable_keys=True, html_wrapper=True, fill_screen=True, height='100%', **display_kwargs) self.set_header("Content-Type", "text/html") self.serve(html)
def __init__(self, pcsd_ruby_socket, debug=False): self.__debug = debug AsyncHTTPClient.configure( "tornado.curl_httpclient.CurlAsyncHTTPClient") self.__client = AsyncHTTPClient() self.__pcsd_ruby_socket = pcsd_ruby_socket
def http_client(self): return AsyncHTTPClient( force_instance=True, defaults=dict(validate_cert=self.tls_verify) )
def get(self): http_client = AsyncHTTPClient() response = yield http_client.fetch("http://www.sinacloud.com") self.set_header('content-type', 'text/plain') self.write('Hello, World! ' + str(response.body[:100]))
async def get(self, provider_prefix, _unescaped_spec): prefix = '/v2/' + provider_prefix spec = self.get_spec_from_request(prefix) spec = spec.rstrip("/") try: self.get_provider(provider_prefix, spec=spec) except HTTPError: raise except Exception as e: app_log.error( "Failed to construct provider for %s/%s", provider_prefix, spec, ) # FIXME: 400 assumes it's the user's fault (?) # maybe we should catch a special InvalidSpecError here raise HTTPError(400, str(e)) provider_spec = f'{provider_prefix}/{spec}' social_desc = f"{SPEC_NAMES[provider_prefix]}: {spec}" nbviewer_url = None if provider_prefix == "gh": # we can only produce an nbviewer URL for github right now nbviewer_url = 'https://nbviewer.jupyter.org/github' org, repo_name, ref = spec.split('/', 2) # NOTE: tornado unquotes query arguments too -> notebooks%2Findex.ipynb becomes notebooks/index.ipynb filepath = self.get_argument('filepath', '').lstrip('/') # Check if we have a JupyterLab + file path, if so then use it for the filepath urlpath = self.get_argument('urlpath', '').lstrip('/') if urlpath.startswith("lab") and "/tree/" in urlpath: filepath = urlpath.split('tree/', 1)[-1] blob_or_tree = 'blob' if filepath else 'tree' nbviewer_url = f'{nbviewer_url}/{org}/{repo_name}/{blob_or_tree}/{ref}/{filepath}' # Check if the nbviewer URL is valid and would display something # useful to the reader, if not we don't show it client = AsyncHTTPClient() # quote any unicode characters in the URL proto, rest = nbviewer_url.split("://") rest = urllib.parse.quote(rest) request = HTTPRequest( proto + "://" + rest, method="HEAD", user_agent="BinderHub", ) response = await client.fetch(request, raise_error=False) if response.code >= 400: nbviewer_url = None self.render_template( "loading.html", base_url=self.settings['base_url'], badge_base_url=self.get_badge_base_url(), provider_spec=provider_spec, social_desc=social_desc, nbviewer_url=nbviewer_url, # urlpath=self.get_argument('urlpath', None), submit=True, google_analytics_code=self.settings['google_analytics_code'], google_analytics_domain=self.settings['google_analytics_domain'], extra_footer_scripts=self.settings['extra_footer_scripts'], )
def get_app_http_client(self): AsyncHTTPClient.configure( 'tornado.curl_httpclient.CurlAsyncHTTPClient') return AsyncHTTPClient(force_instance=True)
def post(self, sessionid): rclient = self.settings['rclient'] gen_log.info('sessionid ' + str(sessionid)) if not sessionid: self.render("sorry.html", reason="Invalid Session. This link is not valid") return session = rclient.get(sessionid) if not session: self.render("sorry.html", reason="Invalid Session. This link is not valid") return session = pickle.loads(session) if not session: rclient.delete(sessionid) self.render("sorry.html", reason="Invalid Session. This link is not valid") return inbounddb = self.settings['inbounddb'] user = yield inbounddb.users.find_one({'actual': session['actual']}) sup_user = yield inbounddb.suspended_users.find_one( {'actual': session['actual']}) if not user and not sup_user: rclient.delete(sessionid) self.render("sorry.html", reason="Invalid Attempt. This link is not valid") return otp = self.get_argument('otp', 'junk') http_client = AsyncHTTPClient() response = yield http_client.fetch( "https://cognalys.com/api/v1/otp/confirm/?app_id=" + self.settings['coganlys_app_id'] + "&access_token=" + self.settings['cognalys_acc_token'] + "&keymatch=" + session['keymatch'] + "&otp=" + session['otpstart'] + otp, raise_error=False) if response.code != 200: self.render("sorry.html", reason="Invalid OTP. Verification Failed") rclient.delete(sessionid) return resdata = json.loads(response.body.decode()) gen_log.info('coganlys verify response data ' + str(resdata)) if resdata['status'] != 'success': self.render("sorry.html", reason="Invalid OTP. Verification Failed") rclient.delete(sessionid) return reason = "Thank You for verifing phone number.\n" if session.get('user_data', None): ud = pickle.loads(session['user_data']['sud']) ud['suspended'] = 'False' ud['verify_count'] = 0 ud['phone_verified'] = 'True' yield inbounddb.users.insert(ud) if session['activation'] == 'True': reason += 'Your account has been activated\n' yield inbounddb.suspended_users.remove( {'actual': ud['actual']}) else: yield inbounddb.users.update({'actual': session['actual']}, {'$set': { 'phone_verified': 'True' }}) yield inbounddb.users.update({'actual': session['actual']}, {"$set": { 'verify_count': 0 }}) self.render("success.html", reason=reason) rclient.delete(sessionid) return
if app["domain"].strip() == "": continue domain = app["domain"].strip() dt = datetime.date.today() today = datetime.datetime(dt.year, dt.month, dt.day) r = alexa_collection.find_one(({"domain": domain, "date": today})) if r is not None: continue total += 1 url = "http://www.alexa.com/siteinfo/%s" % app["domain"] request(url, lambda r, app=app: handle_alexa_com_result(r, app)) total += 1 url = "http://www.alexa.cn/index.php?url=%s" % app["domain"] request(url, lambda r, app=app: handle_alexa_cn_result(r, app)) flag = True conn.close() if __name__ == "__main__": logger.info("Start...") AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") http_client = AsyncHTTPClient(max_clients=50) instance = tornado.ioloop.IOLoop.instance() begin() instance.start()
def get_app_http_client(self): AsyncHTTPClient.configure( 'tornado.simple_httpclient.SimpleAsyncHTTPClient') return AsyncHTTPClient(force_instance=True)
def fetch_html(url): http_cli = AsyncHTTPClient() respone = yield http_cli.fetch(url) return respone
def httpclient_instance(self): return AsyncHTTPClient()
def _create_user_name(self, user_uuid=None, data_uuid=None, ip=None): logging.info("create anonymous user_uuid: %s, ip: %s" % (user_uuid, ip)) if user_uuid == None or ip == None or data_uuid == None: return url = "http://123.57.154.168:8099/IP2GEO/" http_headers = {"Content-Type": "application/json"} http_body = { "ip": ip, "language": _get_config().get("server").get("language").get("locale"), "team_uuid": _get_config().get("team").get("app_uuid"), "team_name": _get_config().get("team").get("name") } http_request = HTTPRequest(url, method='POST', headers=http_headers, validate_cert=False, body=json.dumps(http_body)) http_client = AsyncHTTPClient() response = yield http_client.fetch(http_request) logging.info("geoservice return: %s" % response.body) _body = json.loads(response.body) if _body == None or _body.get("error_code") != 0: logging.error("cant get user name by ip: %s" % ip) return _country = _body.get("country") _state = _body.get("state") _city = _body.get("city") _location_user = [] if _country != None and len(_country) != 0: _location_user.append(_country) if _state != None and len(_state) != 0: _location_user.append(_state) if _city != None and len(_city) != 0: _location_user.append(_city) if len(_location_user) == 0: return _user_name = ".".join(_location_user) _row = DeviceUser(uuid=user_uuid, user_name=_user_name, user_fullname=_user_name) _row.update_redis_keys(self.application.redis) _row.async_update(self.application.redis) _row = AppUserData(uuid=data_uuid, user_fullname=_user_name) _row.update_redis_keys(self.application.redis) _row.async_update(self.application.redis) return
def __init__(self, application, request, **kwargs): super().__init__(application, request, **kwargs) self.client = AsyncHTTPClient()
async def post(cls, url, params=None, body=None, headers=None, encode_type='utf-8', decode_type='utf-8', parse_json=True, timeout=30): """ HTTP POST 请求 @param url 请求url @param params 请求的uri qurey参数 @param body 请求的body参数 @param headers 请求的header参数 @param encode_type 请求body编码格式,默认使用utf-8编码 @param decode_type 返回body解码格式,默认使用utf-8解码 @param parse_json 是否解析返回body为json格式,默认为True @param timeout 请求超时时间,默认30秒 @return data 返回的http body """ if params: url = url_concat(url, params) if body: if not encode_type: pass elif encode_type == 'utf-8': body = json.dumps(body) else: body = urlencode(body, encoding=encode_type) http_client = AsyncHTTPClient() response = await http_client.fetch(url, method='POST', body=body, headers=headers, request_timeout=timeout) if response.code not in (200, 201, 202, 203, 204, 205, 206): logger.error('url:', url, 'post data:', body, 'response code:', response.code, 'response body:', response.body, caller=cls) msg = '请求url失败: {url}'.format(url=url) raise exceptions.CustomException(msg=msg) if response.body: data = response.body if decode_type: data = data.decode(decode_type) if parse_json: return json.loads(data) else: return data else: return None
async def get_lms_access_token(token_endpoint: str, private_key_path: str, client_id: str, scope=None) -> str: """ Gets an access token from the LMS Token endpoint by using the private key (pem format) and client id Args: token_endpoint: The url that will be used to make the request private_key_path: specify where the pem is client_id: For LTI 1.3 the Client ID that was obtained with the tool setup Returns: A json with the token value """ token_params = { "iss": client_id, "sub": client_id, "aud": token_endpoint, "iat": int(time.time()) - 5, "exp": int(time.time()) + 60, "jti": str(uuid.uuid4()), } logger.debug("Getting lms access token with parameters %s" % token_params) # get the pem-encoded content private_key = get_pem_text_from_file(private_key_path) headers = get_headers_to_jwt_encode(private_key) token = jwt.encode(token_params, private_key, algorithm="RS256", headers=headers) logger.debug("Obtaining token %s" % token) scope = scope or " ".join([ "https://purl.imsglobal.org/spec/lti-ags/scope/score", "https://purl.imsglobal.org/spec/lti-ags/scope/lineitem", "https://purl.imsglobal.org/spec/lti-ags/scope/result.readonly", "https://purl.imsglobal.org/spec/lti-ags/scope/lineitem.readonly", ]) logger.debug("Scope is %s" % scope) params = { "grant_type": "client_credentials", "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", "client_assertion": token.decode(), "scope": scope, } logger.debug("OAuth parameters are %s" % params) client = AsyncHTTPClient() body = urllib.parse.urlencode(params) try: resp = await client.fetch(token_endpoint, method="POST", body=body, headers=None) except HTTPClientError as e: logger.info( f"Error by obtaining a token with lms. Detail: {e.response.body if e.response else e.message}" ) raise logger.debug("Token response body is %s" % json.loads(resp.body)) return json.loads(resp.body)
def async_http(url): '''Fetch a URL asynchronously''' httpclient = AsyncHTTPClient() result = yield httpclient.fetch(url) raise gen.Return(result.body)
def get(self): self.log.info('AUTH.') user = yield self.get_current_user() token = self.get_argument('gcube-token') if user and token: self.log.info('Clearing login cookie, new user?') self.clear_login_cookie() # make sure we don't do a mess here self.redirect(url_concat( self.authenticator.login_url(self.hub.base_url), {'gcube-token': token}), permanent=False) return if not token: self.log.error('No gcube token. Out!') raise web.HTTPError(403) http_client = AsyncHTTPClient() # discover user info user_url = url_concat( url_path_join(D4SCIENCE_SOCIAL_URL, D4SCIENCE_PROFILE), {'gcube-token': token}) req = HTTPRequest(user_url, method='GET') try: resp = yield http_client.fetch(req) except HTTPError as e: # whatever, get out self.log.warning('Something happened with gcube service: %s', e) raise web.HTTPError(403) resp_json = json.loads(resp.body.decode('utf8', 'replace')) username = resp_json.get('result', {}).get('username', '') if not username: self.log.error('Unable to get the user from gcube?') raise web.HTTPError(403) # discover WPS self.log.info('Discover wps') wps_endpoint = '' discovery_url = url_concat(D4SCIENCE_DM_REGISTRY_URL, {'gcube-token': token}) req = HTTPRequest(discovery_url, method='GET') try: self.log.info('fetch') resp = yield http_client.fetch(req) except HTTPError as e: # whatever, get out self.log.warning('Something happened with gcube service: %s', e) raise web.HTTPError(403) root = ElementTree.fromstring(resp.body.decode('utf8', 'replace')) self.log.info('root %s', root) for child in root.findall('Resource/Profile/AccessPoint/' 'Interface/Endpoint'): entry_name = child.attrib["EntryName"] self.log.info('entry_name %s', entry_name) if entry_name != "GetCapabilities": wps_endpoint = child.text self.log.info('WPS endpoint: %s', wps_endpoint) break self.log.info('D4Science user is %s', username) self.log.info('WPS %s', wps_endpoint) data = { 'gcube-token': token, 'gcube-user': username, 'wps-endpoint': wps_endpoint } data.update(resp_json['result']) user = yield self.login_user(data) if user: self._jupyterhub_user = user self.redirect(self.get_next_url(user), permanent=False)
def async_http2(url1, url2): '''Fetch two URLs asynchronously''' httpclient = AsyncHTTPClient() r1, r2 = yield [httpclient.fetch(url1), httpclient.fetch(url2)] raise gen.Return(r1.body + r2.body)
async def get(self, provider_prefix, _unescaped_spec): prefix = "/v2/" + provider_prefix spec = self.get_spec_from_request(prefix) spec = spec.rstrip("/") try: self.get_provider(provider_prefix, spec=spec) except HTTPError: raise except Exception as e: app_log.error( "Failed to construct provider for %s/%s", provider_prefix, spec, ) # FIXME: 400 assumes it's the user's fault (?) # maybe we should catch a special InvalidSpecError here raise HTTPError(400, str(e)) provider_spec = f"{provider_prefix}/{spec}" social_desc = f"{SPEC_NAMES[provider_prefix]}: {spec}" nbviewer_url = None if provider_prefix == "gh": # We can only produce an nbviewer URL for github right now nbviewer_url = "https://nbviewer.jupyter.org/github" org, repo_name, ref = spec.split("/", 2) # NOTE: tornado unquotes query arguments too -> notebooks%2Findex.ipynb becomes notebooks/index.ipynb filepath = self.get_argument("labpath", "").lstrip("/") if not filepath: filepath = self.get_argument("filepath", "").lstrip("/") # Check the urlpath parameter for a file path, if so use it for the filepath urlpath = self.get_argument("urlpath", "").lstrip("/") if urlpath and "/tree/" in urlpath: filepath = urlpath.split("tree/", 1)[-1] blob_or_tree = "blob" if filepath else "tree" nbviewer_url = ( f"{nbviewer_url}/{org}/{repo_name}/{blob_or_tree}/{ref}/{filepath}" ) # Check if the nbviewer URL is valid and would display something # useful to the reader, if not we don't show it client = AsyncHTTPClient() # quote any unicode characters in the URL proto, rest = nbviewer_url.split("://") rest = urllib.parse.quote(rest) request = HTTPRequest( proto + "://" + rest, method="HEAD", user_agent="BinderHub", ) response = await client.fetch(request, raise_error=False) if response.code >= 400: nbviewer_url = None build_token = jwt.encode( { "exp": int(time.time()) + self.settings["build_token_expires_seconds"], "aud": provider_spec, "origin": self.token_origin(), }, key=self.settings["build_token_secret"], algorithm="HS256", ) self.render_template( "loading.html", base_url=self.settings["base_url"], badge_base_url=self.get_badge_base_url(), build_token=build_token, provider_spec=provider_spec, social_desc=social_desc, nbviewer_url=nbviewer_url, # urlpath=self.get_argument('urlpath', None), submit=True, google_analytics_code=self.settings["google_analytics_code"], google_analytics_domain=self.settings["google_analytics_domain"], extra_footer_scripts=self.settings["extra_footer_scripts"], )
def get_testerhome(self): http_client = AsyncHTTPClient() response = yield http_client.fetch("https://testerhome.com/") raise gen.Return(response.body)
def cull_idle(url, api_token, inactive_limit, cull_users=False, max_age=0, concurrency=10): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ auth_header = { 'Authorization': 'token %s' % api_token, } req = HTTPRequest( url=url + '/users', headers=auth_header, ) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch resp = yield fetch(req) users = json.loads(resp.body.decode('utf8', 'replace')) futures = [] @coroutine def handle_server(user, server_name, server): """Handle (maybe) culling a single server Returns True if server is now stopped (user removable), False otherwise. """ log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning( "Not culling server %s with pending %s", log_name, server['pending']) return False # jupyterhub < 0.9 defined 'server.url' once the server was ready # as an *implicit* signal that the server was ready. # 0.9 adds a dedicated, explicit 'ready' field. # By current (0.9) definitions, servers that have no pending # events and are not ready shouldn't be in the model, # but let's check just to be safe. if not server.get('ready', bool(server['url'])): app_log.warning( "Not culling not-ready not-pending server %s: %s", log_name, server) return False if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if server['last_activity']: inactive = now - parse_date(server['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info( "Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug( "Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) return False if server_name: # culling a named server delete_url = url + "/users/%s/servers/%s" % ( quote(user['name']), quote(server['name']) ) else: delete_url = url + '/users/%s/server' % quote(user['name']) req = HTTPRequest( url=delete_url, method='DELETE', headers=auth_header, ) resp = yield fetch(req) if resp.code == 202: app_log.warning( "Server %s is slow to stop", log_name, ) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if 'servers' in user: servers = user['servers'] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user['server']: servers[''] = { 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug( "Not culling user %s with %i servers still alive", user['name'], still_alive) return False should_cull = False if user.get('created'): age = now - parse_date(user['created']) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user['last_activity']: inactive = now - parse_date(user['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info( "Culling user %s (inactive for %s)", user['name'], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling user %s (age: %s, inactive for %s)", user['name'], format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug( "Not culling user %s (created: %s, last active: %s)", user['name'], format_td(age), format_td(inactive)) return False req = HTTPRequest( url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header, ) yield fetch(req) return True for user in users: futures.append((user['name'], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
def get_http_client(self): return AsyncHTTPClient(io_loop=self.io_loop)
from urllib.parse import urlencode, quote from tornado import gen from tornado.httpclient import AsyncHTTPClient from tornado.web import RequestHandler, Application from tornado.ioloop import IOLoop from apscheduler.schedulers.tornado import TornadoScheduler parent_path = os.path.dirname(sys.path[0]) if parent_path not in sys.path: sys.path.append(parent_path) from fileBasedConfiguration import ApplicationProperties from constants import RequestMapping http_client = AsyncHTTPClient() http_header = { #"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36" "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) VAR/1.0.0.1" } # (ip, port, expire, used_times) global_ip_pool = [([""] * 4) for i in range(1)] global_ip_dic = {} ip_dic_lock = threading.Lock() async def refreship(): maxtimes = int( ApplicationProperties.configure(
def cull_idle( url, api_token, inactive_limit, cull_users=False, remove_named_servers=False, max_age=0, concurrency=10, ssl_enabled=False, internal_certs_location="", ): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ if ssl_enabled: ssl_context = make_ssl_context( f"{internal_certs_location}/hub-internal/hub-internal.key", f"{internal_certs_location}/hub-internal/hub-internal.crt", f"{internal_certs_location}/hub-ca/hub-ca.crt", ) app_log.debug("ssl_enabled is Enabled: %s", ssl_enabled) app_log.debug("internal_certs_location is %s", internal_certs_location) AsyncHTTPClient.configure(None, defaults={"ssl_options": ssl_context}) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch # Starting with jupyterhub 1.3.0 the users can be filtered in the server # using the `state` filter parameter. "ready" means all users who have any # ready servers (running, not pending). auth_header = {"Authorization": "token %s" % api_token} resp = yield fetch(HTTPRequest(url=url + "/info", headers=auth_header)) info = json.loads(resp.body.decode("utf8", "replace")) state_filter = V(info["version"]) >= STATE_FILTER_MIN_VERSION req = HTTPRequest( url=url + "/users%s" % ("?state=ready" if state_filter else ""), headers=auth_header, ) now = datetime.now(timezone.utc) resp = yield fetch(req) users = json.loads(resp.body.decode("utf8", "replace")) app_log.debug("Got %d users%s", len(users), (" with ready servers" if state_filter else "")) futures = [] @coroutine def handle_server(user, server_name, server, max_age, inactive_limit): """Handle (maybe) culling a single server "server" is the entire server model from the API. Returns True if server is now stopped (user removable), False otherwise. """ log_name = user["name"] if server_name: log_name = "%s/%s" % (user["name"], server_name) if server.get("pending"): app_log.warning("Not culling server %s with pending %s", log_name, server["pending"]) return False # jupyterhub < 0.9 defined 'server.url' once the server was ready # as an *implicit* signal that the server was ready. # 0.9 adds a dedicated, explicit 'ready' field. # By current (0.9) definitions, servers that have no pending # events and are not ready shouldn't be in the model, # but let's check just to be safe. if not server.get("ready", bool(server["url"])): app_log.warning("Not culling not-ready not-pending server %s: %s", log_name, server) return False if server.get("started"): age = now - parse_date(server["started"]) else: # started may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if server["last_activity"]: inactive = now - parse_date(server["last_activity"]) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age # CUSTOM CULLING TEST CODE HERE # Add in additional server tests here. Return False to mean "don't # cull", True means "cull immediately", or, for example, update some # other variables like inactive_limit. # # Here, server['state'] is the result of the get_state method # on the spawner. This does *not* contain the below by # default, you may have to modify your spawner to make this # work. The `user` variable is the user model from the API. # # if server['state']['profile_name'] == 'unlimited' # return False # inactive_limit = server['state']['culltime'] should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive), ) return False body = None if server_name: # culling a named server # A named server can be stopped and kept available to the user # for starting again or stopped and removed. To remove the named # server we have to pass an additional option in the body of our # DELETE request. delete_url = url + "/users/%s/servers/%s" % ( quote(user["name"]), quote(server["name"]), ) if remove_named_servers: body = json.dumps({"remove": True}) else: delete_url = url + "/users/%s/server" % quote(user["name"]) req = HTTPRequest( url=delete_url, method="DELETE", headers=auth_header, body=body, allow_nonstandard_methods=True, ) resp = yield fetch(req) if resp.code == 202: app_log.warning("Server %s is slow to stop", log_name) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if "servers" in user: servers = user["servers"] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user["server"]: servers[""] = { "last_activity": user["last_activity"], "pending": user["pending"], "url": user["server"], } server_futures = [ handle_server(user, server_name, server, max_age, inactive_limit) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug( "Not culling user %s with %i servers still alive", user["name"], still_alive, ) return False should_cull = False if user.get("created"): age = now - parse_date(user["created"]) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user["last_activity"]: inactive = now - parse_date(user["last_activity"]) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling user %s (inactive for %s)", user["name"], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling user %s (age: %s, inactive for %s)", user["name"], format_td(age), format_td(inactive), ) should_cull = True if not should_cull: app_log.debug( "Not culling user %s (created: %s, last active: %s)", user["name"], format_td(age), format_td(inactive), ) return False req = HTTPRequest(url=url + "/users/%s" % user["name"], method="DELETE", headers=auth_header) yield fetch(req) return True for user in users: futures.append((user["name"], handle_user(user))) # If we filtered users by state=ready then we did not get back any which # are inactive, so if we're also culling users get the set of users which # are inactive and see if they should be culled as well. if state_filter and cull_users: req = HTTPRequest(url=url + "/users?state=inactive", headers=auth_header) resp = yield fetch(req) users = json.loads(resp.body.decode("utf8", "replace")) app_log.debug("Got %d users with inactive servers", len(users)) for user in users: futures.append((user["name"], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
def get_http_client(self): return AsyncHTTPClient(force_instance=True, defaults=dict(validate_cert=False))
def get(self): io_loop = self.request.connection.stream.io_loop client = AsyncHTTPClient(io_loop=io_loop) response = yield gen.Task(client.fetch, self.get_argument('url')) response.rethrow() self.finish(b("got response: ") + response.body)
def get_http_client(self): return AsyncHTTPClient()
def up_yflow(handler, partner): handler.up_req_time = time.localtime() confId = partner["confId"] callback = partner["callback"] apikey = partner["apikey"] mobile = handler.mobile seqNo = handler.order_id t = int(time.time()) data = apikey + str(confId) + mobile + str(t) sign = to_md5(data) flow = None k = 'private:yflow:{carrier}:{price}'.format(carrier=handler.carrier, price=handler.price) flow = handler.slave.get(k) flow = int(flow) if flow is None: handler.up_result = 5003 return handler.up_result body = { "confId": confId, "mobile": mobile, "flow": flow, "callback": callback, "t": t, "sign": sign, "seqNo": seqNo, } body = urlencode(body) url = partner["url_busi"] url = url + '?' + body result = 9999 http_client = AsyncHTTPClient() try: request_log.info("REQU %s", body, extra={'orderid': handler.order_id}) response = yield http_client.fetch(url, method='GET', request_timeout=120) except HTTPError as http_error: request_log.error('CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id}) result = 60000 + http_error.code response = None except Exception as e: request_log.error('CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id}) response = None finally: http_client.close() handler.up_resp_time = time.localtime() if response and response.code == 200: response_body = response.body.decode('utf8') request_log.info("RESP %s", response_body, extra={'orderid': handler.order_id}) try: response_body = json.loads(response_body) retCode = response_body["retCode"] result = RESULT_MAP.get(retCode, 9) handler.up_result = str(result) except Exception as e: result = 9999 handler.up_result = result request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id}) return result