def fetch(self, url, **kwargs): # init HTTPRequest session = HTTPRequest('', follow_redirects=False) self.init_request(session, url, **kwargs) instance_parameters = copy.deepcopy(self._req_params) # 参数 http_client = AsyncHTTPClient() while True: self.pre_request(session, url, **kwargs) try: response = yield http_client.fetch(session, **instance_parameters) break except HTTPError as httperr: # redirects handler if httperr.code > 300 and httperr.code < 400: self.post_request(session, httperr.response, url, **kwargs) session.url = httperr.response.effective_url del instance_parameters self.post_request(session, response, url, **kwargs) raise gen.Return(response)
def open_root_url(step): request = HTTPRequest(ROOT_URL + 'user/state/') if world.cookie: request.headers["Cookie"] = world.cookie resp = world.browser.fetch(request) world.response = json_decode(resp.body) world.response_header = resp.headers
def get_request(self, url, method='GET', body=None, headers=None): url = self.get_url(url) req = HTTPRequest(url, method=method, body=body) req.headers['Content-Type'] = 'application/json; charset=UTF-8' if headers: req.headers.update(headers) return req
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) if not isinstance(request.headers, HTTPHeaders): request.headers = HTTPHeaders(request.headers) callback = stack_context.wrap(callback) _HTTPConnection(self.io_loop, request, callback)
def _create_http_request(self, method, host, port, path, params=None, data=None, **kwargs): url = 'http://{host}:{port}{uri}'.format(host=host, port=port, uri=path) if params and isinstance(params, dict): url += '?' + urlencode(params) request = HTTPRequest( method=method, url=url, allow_nonstandard_methods=True, connect_timeout=self._connect_timeout, request_timeout=self._request_timeout, **kwargs ) if data and method in ['POST', 'PUT', 'PATCH']: try: request.body = json.dumps(data) except TypeError as e: logging.error(str(e)) raise DBApiError(e) return request
def getData(self,url,method,data,cookie): try: client = HTTPClient() request = HTTPRequest( url, method=method, headers={ 'Cookie':cookie } ) if data and method=="GET": url = url_concat(url,data) url = url.replace("+","%20") request.url = url elif data and method=="POST": realData = {} for i in data: realData[i[0]] = i[1] data = urllib.urlencode(realData) request.body = data response = client.fetch(request) return json.loads(response.body) except Exception,e: # print str(e) #traceback.print_exc() return str(e)
def getData(self,url,method,data,cookie): try: client = HTTPClient() request = HTTPRequest( url, method=method, headers={ 'Cookie':cookie } ) if data and method=="GET": data = json.loads(data) url = url_concat(url,data) request.url = url elif data and method=="POST": data = json.loads(data) print data data = urllib.urlencode(data) request.body = data # print request.url response = client.fetch(request) return response.body except Exception,e: # print str(e) return None
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We're going to modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) callback = stack_context.wrap(callback) key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( time.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key)) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: logging.debug( 'max_clients limit reached, request queued. ' '%d active, %d queued requests.' % ( len(self.active), len(self.queue)) )
def perform_request(self, request, response, method): try: constants = request_consts[method] url = request[constants.URL] timeout = request[constants.TIMEOUT] http_request = HTTPRequest(url=url, method=method) http_request.request_timeout = float(timeout)/1000 if method == 'POST': http_request.body = request[constants.BODY] #adds cookies to request params_num = len(request) if constants.COOKIES <= params_num - 1: cookies = request[constants.COOKIES] if len(cookies) > 0: list_of_cookies = list('{0}={1}'.format(cookie, value) for cookie, value in cookies.iteritems()) cookies_str = '; '.join(list_of_cookies) http_request.headers.add('Cookie', cookies_str) #adds headers to request if constants.HEADERS <= params_num - 1: for name, values_list in request[constants.HEADERS].iteritems(): for value in values_list: http_request.headers.add(name, value) self.logger.info("Downloading {0}, headers {1}, method {2}".format(url, http_request.headers, method)) http_response = yield self.http_client.fetch(http_request) response_headers = self._get_headers_from_response(http_response) response.write((True, http_response.body, http_response.code, response_headers,)) response.close() self.logger.info("{0} has been successfuly downloaded".format(url)) except HTTPError as e: self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url)) if e.response is not None: http_response = e.response response_headers = self._get_headers_from_response(http_response) response.write((False, http_response.body, http_response.code, response_headers,)) else: response.write((False, '', e.code, {},)) response.close() except socket.gaierror as e: self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url)) response.write((False, '', e.errno, {},)) response.close() except Exception as e: self.logger.error("Unhandled error ({0}) occured in perform_request, report about this problem " "to httpclient service developers. Method is {1}, stacktrace is: {2}".format( e.message, method, traceback.format_exc())) response.write((False, '', 0, {},)) response.close()
def create_request(self): request = HTTPRequest(url=self.request_data['url'], headers=self.request_data['headers']) self.patch_request(request) request.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36' #request.connect_timeout = 0 request.request_timeout = 0 request.header_only = False return request
def fetch(self, path, body=None, **kwargs): kwargs['url'] = self.get_url(path) request = HTTPRequest(**kwargs) if body is not None: request.body = body request.allow_nonstandard_methods = True self.http_client.fetch(request, self.stop, method=None) return self.wait()
def send_logout_request(step): request = HTTPRequest(ROOT_URL + "logout/", validate_cert=False) if world.cookie: request.headers["Cookie"] = world.cookie world.response = world.browser.fetch(request) if world.response.code == 200: world.cookie = "" world.browser.cookie = ""
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) if not isinstance(request.headers, HTTPHeaders): request.headers = HTTPHeaders(request.headers) callback = stack_context.wrap(callback) self.queue.append((request, callback)) self._process_queue()
def fetch(self, path, **kwargs): kwargs['url'] = self.get_url(path) body = kwargs.pop('body', None) request = HTTPRequest(**kwargs) request.body = body request.allow_nonstandard_methods = True request.request_timeout = TIMEOUT self.http_client.fetch(request, self.stop, **kwargs) return self.wait()
def wrapper(self, request, callback, **kwargs): with HttpClientTransContext("tornado.simple_httpclient.SimpleAsyncHTTPClient.fetch") as context: if context: from tornado.httpclient import HTTPRequest if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) request._td_httpclient_node = context return func(self, request, callback, **kwargs)
def post(self): # param from the client message = {} message['status'] = 'error' message['returned_url'] = 'null' #url = self.get_argument('url', None) comic_id = self.get_argument('img', None) self.threash_hold = self.get_argument('threshold', 90) if not self.threash_hold: self.threash_hold = float(90) else: self.threash_hold = float(self.threash_hold) if comic_id: comic_id = int(comic_id[5:]) if len(self.request.files['file']) > 0: img_file = self.request.files['file'][0] # check the client params if not img_file or comic_id <= 0: self.write(json.dumps(message)) self.finish() else: self.comic_picture_id = comic_id ###################################################### # print self.client_cv_img filename = img_file['filename'] saved_path = self.IMAGE_PATH + filename self.client_saved_img = saved_path if not os.path.exists(filename): with open(saved_path, "wb") as fp: fp.write(img_file['body']) # save the client img self.client_cv_img = scv.Image(r'%s' % str(saved_path)) ###################################################### # ok, we save the client image and gen the SimpleCV img # we pass the client img url to oxford to get the params # get parameters get_params = urllib.urlencode({ 'analyzesFaceLandmarks': 'true', 'analyzesAge': 'true', 'analyzesGender': 'true', 'analyzesHeadPose': 'true', }) url = self.API_URL % get_params post_params = {} post_params['url'] = self.HOST + saved_path #print post_params['url'] # request request = HTTPRequest(url, method='POST') request.body = json.dumps(post_params) request.headers['Content-Type'] = 'application/json' request.headers['Ocp-Apim-Subscription-key'] = self.TOKEN # fetch client = AsyncHTTPClient() client.fetch(request, self.handleResponse)
def fetch(self, request, **kwargs): headers = kwargs['headers'] if 'headers' in kwargs else default_headers if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, headers = headers, **kwargs) request.headers = HTTPHeaders(request.headers if request.headers else headers) try: return self.http.fetch(request) except (HTTPError, SyntaxError), e: #todo log print "[ERROR]", e
def fetch(self, url, headers=None, body=None, method="GET", callback=None, raise_error=True, cache=None, **kwargs): headers = headers or {} body = body or "{}" """very simlar with AsyncHTTPClient.fetch """ if self._closed: raise RuntimeError("fetch() called on closed AsyncHTTPClient") future = TracebackFuture() if isinstance(body, dict): for k,v in body.items(): if v is None: del body[k] body = urllib.urlencode(body) for k,v in headers.items(): #headers 只能接收str if v: headers[k] = str(headers[k]) else: del headers[k] request = HTTPRequest(url=url,method=method,headers=headers,body=body, allow_nonstandard_methods=True, request_timeout=600 ,**kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if raise_error and response.error: future.set_exception(response.error) else: try: resp = json.loads(str(response.body)) if resp.get("statusCode") and resp.get("statusCode")==800: future.set_result(resp) log.info(json.dumps({"response":resp,"body":body,"headers":headers,"url":url})) else: future.set_result({"error_type":"statusCode is not 800", "response":resp,"body":body,"headers":headers,"url":url}) log.error(json.dumps({"error_type":"statusCode is not 800", "response":resp,"body":body,"headers":headers,"url":url})) except Exception,e: future.set_result({"error_type":"json.loads failed!","error":str(e),"response.body":response.body,"body":body,"headers":headers,"url":url}) log.error(json.dumps({"error_type":"json.loads failed!","error":str(e),"response.body":response.body,"body":body,"headers":headers,"url":url}))
def _request(self, extension, body, callback): """Wraps tornado syntax for sending an HTTP request.""" request = HTTPRequest('http://{}/{}'.format(self.ip, extension), connect_timeout=self.timeout, request_timeout=self.timeout) if body: request.body = body request.method = 'POST' if extension == 'ToolWeb/Cmd': request.headers = {'Content-Type': 'text/xml'} self.client.fetch(request, callback)
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We're going to modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) # Early prepare request = _HTTP2Stream.prepare_request(request, self.host) super(SimpleAsyncHTTP2Client, self).fetch(request, callback, **kwargs)
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) if not isinstance(request.headers, HTTPHeaders): request.headers = HTTPHeaders(request.headers) callback = stack_context.wrap(callback) self.queue.append((request, callback)) self._process_queue() if self.queue: logging.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % ( len(self.active), len(self.queue)))
def post(self, url, body): """ Perform a POST request. """ if hasattr(self, "root_url") and self.root_url: url = self.root_url + url request = HTTPRequest(url, method="POST", body=body, validate_cert=False) if hasattr(self, "cookie") and self.cookie: request.headers["Cookie"] = self.cookie return HTTPClient.fetch(self, request)
def delete(self, url): """ Perform a DELETE request. """ if hasattr(self, "root_url") and self.root_url: url = self.root_url + url request = HTTPRequest(url, method="DELETE", validate_cert=False) if self.cookie: request.headers["Cookie"] = self.cookie return HTTPClient.fetch(self, request)
def get(self, url): """ Perform a GET request. """ if hasattr(self, "root_url") and self.root_url: url = self.root_url + url request = HTTPRequest(url, validate_cert=False) if hasattr(self, "cookie") and self.cookie: request.headers["Cookie"] = self.cookie return HTTPClient.fetch(self, request)
def do_send_sms(cell, data): http_client = HTTPClient() sms_url = "http://172.31.11.203:8080/notify/sms/" request = HTTPRequest(sms_url) request.headers["Content-Type"] = "application/json" request.headers["HTTP_HEAD_ENCODING"] = "utf-8" request.method = "POST" request.body = '{"id":"0","phones":"' + cell + '","content":"' + data + '"}' resp = http_client.fetch(request) if resp.code != 201: raise RuntimeError("SMS Gateway Error: " + str(resp.code)) print resp.code, resp.body
def post_a_new_picture_via_the_dedicated_resource(step): time.sleep(1) file = open("test.jpg", "r") (contentType, body) = encode_multipart_formdata([], [("picture", "test.jpg", file.read())]) headers = {'Content-Type':contentType} request = HTTPRequest(url=world.browser.root_url + "pictures/all/", method="POST", body=body, headers=headers, validate_cert=False) if hasattr(world.browser, "cookie"): request.headers["Cookie"] = world.browser.cookie world.browser.fetch(request)
def send_request(self, method, url, data=None, data_type="json"): method = method.upper() has_payload = method == self.POST or method == self.PUT is_CUD = has_payload or method == self.DELETE full_url = urlparse.urlunparse((controller_address['scheme'], controller_address['host'] + ':' + str(controller_address['port']), url, None, None, None)) headers = { 'Content-Type': HttpClient.set_content_type(data_type) } request = HTTPRequest(url=full_url, method=method, headers=headers, auth_username=controller_auth['username'], auth_password=controller_auth['password'], connect_timeout=http_client_settings.get("timeouts", {}).get("connect", 3), request_timeout=http_client_settings.get("timeouts", {}).get("request", 10)) if has_payload: if data_type == "json": request.body = json.dumps(data) if is_CUD: if self.dry_run: logger.info("\nDRY RUN") logger.debug("\n\nSending {} request.\nUrl: {}\nBody: {}\n".format(method, full_url, request.body)) if is_CUD and self.dry_run: response = json.dumps({ "status": "ok", "msg": "dry_run" }) else: try: response = HTTPClient().fetch(request) if not self.fail_silently and not self.is_ok(response.code): raise HttpClientException(response) logger.debug("\n\nResponse ({}).\nUrl: {}\nBody: {}\n".format(response.code, full_url, response.body)) return response except HTTPError as e: logger.debug("HttpClient error: {}".format(e.message)) if not self.fail_silently: raise HttpClientException(e) return None return response
def fetch(self, request, callback=None, raise_error=True, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an `HTTPResponse`. By default, the ``Future`` will raise an `HTTPError` if the request returned a non-200 response code. Instead, if ``raise_error`` is set to False, the response will always be returned regardless of the response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ if self._closed: raise RuntimeError("fetch() called on closed AsyncHTTPClient") if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if raise_error and response.error: future.set_exception(response.error) else: future.set_result(response) self.fetch_impl(request, handle_response) return future
def _make_proxy_request(self, request_data): timeout = float(request_data.get("timeout", DEFAULT_TIMEOUT)) validate_cert = bool(request_data.get("validate_cert") or True) max_redirects = request_data.get("max_http_redirects") or 0 follow_redirects = max_redirects > 0 # 0 means do not follow redirects url = request_data.get("url") params = request_data.get("data") post_type = request_data.get("post_type") if params and post_type is None: url = "%s?%s" % (url, urlencode(params)) logger.info("[%s]agent request url: %s", self.id, url) proxy_request = HTTPRequest( url, validate_cert=validate_cert, headers=self._get_proxy_request_headers(request_data), method=request_data.get("method", "GET"), allow_nonstandard_methods=True, connect_timeout=timeout, request_timeout=timeout, streaming_callback=self._streaming_callback, header_callback=self._header_callback, follow_redirects=follow_redirects, max_redirects=max_redirects, prepare_curl_callback=self.prepare_curl_callback, ) role_name = request_data.get("role") if role_name: InterfaceRoleManager.set_curl_interface_role( proxy_request, role_name, ) keystone_auth_info = request_data.get("keystone") if keystone_auth_info: logger.warning( "[%s]agent request required keystone token", ) auth_headers = yield self._get_keystone_auth_headers( keystone_auth_info, validate_cert=validate_cert, ) if not auth_headers: raise gen.Return() proxy_request.headers.update(auth_headers) body = self._get_request_body(request_data) if body: proxy_request.body = body raise gen.Return(proxy_request)
def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We're going to modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = HTTPHeaders(request.headers) callback = stack_context.wrap(callback) self.queue.append((request, callback)) self._process_queue() if self.queue: logging.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % ( len(self.active), len(self.queue)))
def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # named servers contain the 'servers' dict if 'servers' in user: servers = user['servers'] # Otherwise, server data is intermingled in with the user # model else: servers = {} if user['server']: servers[''] = { 'started': user.get('started'), 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug("Not culling user %s with %i servers still alive", user['name'], still_alive) return False should_cull = False if user.get('created'): age = now - parse_date(user['created']) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user['last_activity']: inactive = now - parse_date(user['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling user %s (inactive for %s)", user['name'], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info("Culling user %s (age: %s, inactive for %s)", user['name'], format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug("Not culling user %s (created: %s, last active: %s)", user['name'], format_td(age), format_td(inactive)) return False req = HTTPRequest( url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header, ) yield fetch(req) return True
def cull_idle( url, api_token, inactive_limit, cull_users=False, remove_named_servers=False, max_age=0, concurrency=10, ): """Shutdown idle single-user servers If cull_users, inactive *users* will be deleted as well. """ auth_header = { 'Authorization': 'token %s' % api_token, } req = HTTPRequest( url=url + '/users', headers=auth_header, ) now = datetime.now(timezone.utc) client = AsyncHTTPClient() if concurrency: semaphore = Semaphore(concurrency) @coroutine def fetch(req): """client.fetch wrapped in a semaphore to limit concurrency""" yield semaphore.acquire() try: return (yield client.fetch(req)) finally: yield semaphore.release() else: fetch = client.fetch resp = yield fetch(req) users = json.loads(resp.body.decode('utf8', 'replace')) futures = [] @coroutine def handle_server(user, server_name, server): """Handle (maybe) culling a single server Returns True if server is now stopped (user removable), False otherwise. """ log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning("Not culling server %s with pending %s", log_name, server['pending']) return False if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if server['last_activity']: inactive = now - parse_date(server['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info("Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug("Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) return False body = None if server_name: # culling a named server # A named server can be stopped and kept available to the user # for starting again or stopped and removed. To remove the named # server we have to pass an additional option in the body of our # DELETE request. delete_url = url + "/users/%s/servers/%s" % ( quote(user['name']), quote(server['name']), ) if remove_named_servers: body = json.dumps({"remove": True}) else: delete_url = url + '/users/%s/server' % quote(user['name']) req = HTTPRequest( url=delete_url, method='DELETE', headers=auth_header, body=body, allow_nonstandard_methods=True, ) resp = yield fetch(req) if resp.code == 202: app_log.warning( "Server %s is slow to stop", log_name, ) # return False to prevent culling user with pending shutdowns return False return True @coroutine def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # named servers contain the 'servers' dict if 'servers' in user: servers = user['servers'] # Otherwise, server data is intermingled in with the user # model else: servers = {} if user['server']: servers[''] = { 'started': user.get('started'), 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug("Not culling user %s with %i servers still alive", user['name'], still_alive) return False should_cull = False if user.get('created'): age = now - parse_date(user['created']) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user['last_activity']: inactive = now - parse_date(user['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling user %s (inactive for %s)", user['name'], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info("Culling user %s (age: %s, inactive for %s)", user['name'], format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug("Not culling user %s (created: %s, last active: %s)", user['name'], format_td(age), format_td(inactive)) return False req = HTTPRequest( url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header, ) yield fetch(req) return True for user in users: futures.append((user['name'], handle_user(user))) for (name, f) in futures: try: result = yield f except Exception: app_log.exception("Error processing %s", name) else: if result: app_log.debug("Finished culling %s", name)
def authenticate(self, handler, data=None): code = handler.get_argument("code") # TODO: Configure the curl_httpclient for tornado http_client = AsyncHTTPClient() params = dict(redirect_uri=self.get_callback_url(handler), code=code, grant_type='authorization_code') params.update(self.extra_params) if self.token_url: url = self.token_url else: raise ValueError( "Please set the OAUTH2_TOKEN_URL environment variable") b64key = base64.b64encode( bytes("{}:{}".format(self.client_id, self.client_secret), "utf8")) headers = { "Accept": "application/json", "User-Agent": "JupyterHub", "Authorization": "Basic {}".format(b64key.decode("utf8")) } req = HTTPRequest( url, method="POST", headers=headers, validate_cert=self.tls_verify, body=urllib.parse.urlencode( params) # Body is required for a POST... ) resp = yield http_client.fetch(req) resp_json = json.loads(resp.body.decode('utf8', 'replace')) access_token = resp_json['access_token'] refresh_token = resp_json.get('refresh_token', None) token_type = resp_json['token_type'] scope = (resp_json.get('scope', '')).split(' ') # Determine who the logged in user is headers = { "Accept": "application/json", "User-Agent": "JupyterHub", "Authorization": "{} {}".format(token_type, access_token) } if self.userdata_url: url = url_concat(self.userdata_url, self.userdata_params) else: raise ValueError( "Please set the OAUTH2_USERDATA_URL environment variable") req = HTTPRequest(url, method=self.userdata_method, headers=headers, validate_cert=self.tls_verify, body=urllib.parse.urlencode( {'access_token': access_token})) resp = yield http_client.fetch(req) resp_json = json.loads(resp.body.decode('utf8', 'replace')) if not resp_json.get(self.username_key): self.log.error("OAuth user contains no key %s: %s", self.username_key, resp_json) return return { 'name': resp_json.get(self.username_key), 'auth_state': { 'access_token': access_token, 'refresh_token': refresh_token, 'oauth_user': resp_json, 'scope': scope, } }
async def test_facts_paging(server, client, order_by_column, order, env_with_facts): """Test querying facts with paging, using different sorting parameters.""" env, _, _ = env_with_facts result = await client.get_all_facts( env, filter={"name": "res"}, ) assert result.code == 200 assert len(result.result["data"]) == 6 all_facts_in_expected_order = sorted(result.result["data"], key=itemgetter(order_by_column, "id"), reverse=order == "DESC") all_fact_ids_in_expected_order = fact_ids(all_facts_in_expected_order) result = await client.get_all_facts(env, limit=2, sort=f"{order_by_column}.{order}", filter={"name": "res"}) assert result.code == 200 assert len(result.result["data"]) == 2 assert fact_ids( result.result["data"]) == all_fact_ids_in_expected_order[:2] assert result.result["metadata"] == { "total": 6, "before": 0, "after": 4, "page_size": 2 } assert result.result["links"].get("next") is not None assert result.result["links"].get("prev") is None port = get_bind_port() base_url = "http://localhost:%s" % (port, ) http_client = AsyncHTTPClient() # Test link for next page url = f"""{base_url}{result.result["links"]["next"]}""" assert "limit=2" in url assert "filter.name=res" in url request = HTTPRequest( url=url, headers={"X-Inmanta-tid": env}, ) response = await http_client.fetch(request, raise_error=False) assert response.code == 200 response = json.loads(response.body.decode("utf-8")) assert fact_ids(response["data"]) == all_fact_ids_in_expected_order[2:4] assert response["links"].get("prev") is not None assert response["links"].get("next") is not None assert response["metadata"] == { "total": 6, "before": 2, "after": 2, "page_size": 2 } # Test link for next page url = f"""{base_url}{response["links"]["next"]}""" # The filters should be present for the links as well assert "limit=2" in url assert "filter.name=res" in url request = HTTPRequest( url=url, headers={"X-Inmanta-tid": env}, ) response = await http_client.fetch(request, raise_error=False) assert response.code == 200 response = json.loads(response.body.decode("utf-8")) next_page_ids = fact_ids(response["data"]) assert next_page_ids == all_fact_ids_in_expected_order[4:] assert response["links"].get("prev") is not None assert response["links"].get("next") is None assert response["metadata"] == { "total": 6, "before": 4, "after": 0, "page_size": 2 } # Test link for previous page url = f"""{base_url}{response["links"]["prev"]}""" assert "limit=2" in url assert "filter.name=res" in url request = HTTPRequest( url=url, headers={"X-Inmanta-tid": env}, ) response = await http_client.fetch(request, raise_error=False) assert response.code == 200 response = json.loads(response.body.decode("utf-8")) prev_page_ids = fact_ids(response["data"]) assert prev_page_ids == all_fact_ids_in_expected_order[2:4] assert response["links"].get("prev") is not None assert response["links"].get("next") is not None assert response["metadata"] == { "total": 6, "before": 2, "after": 2, "page_size": 2 }
def test_defaults_none(self): proxy = _RequestProxy(HTTPRequest('http://example.com/'), None) self.assertIs(proxy.auth_username, None)
def post(self): retjson = {'code': 200, 'content': ''} cardnum = self.get_argument('cardnum', default=None) password = self.get_argument('password', default=None) if not (cardnum and password): retjson['code'] = 400 retjson['content'] = 'params lack' else: # read from cache try: status = self.db.query(PeDetailCache).filter( PeDetailCache.cardnum == cardnum).one() if int(strftime('%H', localtime( time()))) < 8 or (status.date > int(time()) - 10000 and status.text != '*'): if status.text == '*': retjson['content'] = [] self.write( json.dumps(retjson, ensure_ascii=False, indent=2)) self.finish() return self.write(base64.b64decode(status.text)) self.finish() return except NoResultFound: status = PeDetailCache(cardnum=cardnum, text='*', date=int(time())) self.db.add(status) try: self.db.commit() except: self.db.rollback() try: client = AsyncHTTPClient() data = { "Login.Token1": cardnum, "Login.Token2": password, 'goto': "http://mynew.seu.edu.cn/loginSuccess.portal", 'gotoOnFail': "http://mynew.seu.edu.cn/loginFailure.portal" } data1 = { 'IDToken0': '', 'IDToken1': cardnum, 'IDToken2': password, 'IDButton': 'Submit', 'goto': 'http://zccx.seu.edu.cn/', 'gx_charset': 'gb2312' } cookie1 = '' request = HTTPRequest(loginurl1, method='POST', body=urllib.urlencode(data1), follow_redirects=False) initcookie = '' try: response = yield client.fetch(request) except HTTPError as e: initcookie = e.response.headers['Set-Cookie'] init_cookie1 = initcookie.split(';')[4].split(',')[ 1] #+initcookie.split(';')[0] header = { 'Host': 'zccx.seu.edu.cn', 'Accept': 'textml,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36', 'Referer': 'http://zccx.seu.edu.cn/', 'Connection': 'Keep-alive', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.8', 'Cookie': init_cookie1 + ';' + cookie1 + ';' + ';amblcookie=02' } request = HTTPRequest(runurl, method='GET', headers=header) response = yield client.fetch(request) cookie1 = response.headers['Set-Cookie'] header[ 'Cookie'] = init_cookie1 + ';' + cookie1 + ';' + ';amblcookie=02' getpeurl = "http://zccx.seu.edu.cn/SportWeb/gym/gymExercise/gymExercise_query_result_2.jsp?xh=%s" % ( cardnum) request = HTTPRequest(getpeurl, headers=header, request_timeout=8) response = yield client.fetch(request) spider = RunningParser() spider.getRunningTable(response.body) retjson['content'] = spider.table except Exception, e: retjson['code'] = 500 retjson['content'] = str(e)
def _curl_setup_request( self, curl: pycurl.Curl, request: HTTPRequest, buffer: BytesIO, headers: httputil.HTTPHeaders, ) -> None: curl.setopt(pycurl.URL, native_str(request.url)) # libcurl's magic "Expect: 100-continue" behavior causes delays # with servers that don't support it (which include, among others, # Google's OpenID endpoint). Additionally, this behavior has # a bug in conjunction with the curl_multi_socket_action API # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), # which increases the delays. It's more trouble than it's worth, # so just turn off the feature (yes, setting Expect: to an empty # value is the official way to disable this) if "Expect" not in request.headers: request.headers["Expect"] = "" # libcurl adds Pragma: no-cache by default; disable that too if "Pragma" not in request.headers: request.headers["Pragma"] = "" curl.setopt( pycurl.HTTPHEADER, [ "%s: %s" % (native_str(k), native_str(v)) for k, v in request.headers.get_all() ], ) curl.setopt( pycurl.HEADERFUNCTION, functools.partial(self._curl_header_callback, headers, request.header_callback), ) if request.streaming_callback: def write_function(b: Union[bytes, bytearray]) -> int: assert request.streaming_callback is not None self.io_loop.add_callback(request.streaming_callback, b) return len(b) else: write_function = buffer.write curl.setopt(pycurl.WRITEFUNCTION, write_function) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) assert request.connect_timeout is not None curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) assert request.request_timeout is not None curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) if request.user_agent: curl.setopt(pycurl.USERAGENT, native_str(request.user_agent)) else: curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") if request.network_interface: curl.setopt(pycurl.INTERFACE, request.network_interface) if request.decompress_response: curl.setopt(pycurl.ENCODING, "gzip,deflate") else: curl.setopt(pycurl.ENCODING, None) if request.proxy_host and request.proxy_port: curl.setopt(pycurl.PROXY, request.proxy_host) curl.setopt(pycurl.PROXYPORT, request.proxy_port) if request.proxy_username: assert request.proxy_password is not None credentials = httputil.encode_username_password( request.proxy_username, request.proxy_password) curl.setopt(pycurl.PROXYUSERPWD, credentials) if request.proxy_auth_mode is None or request.proxy_auth_mode == "basic": curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_BASIC) elif request.proxy_auth_mode == "digest": curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_DIGEST) else: raise ValueError("Unsupported proxy_auth_mode %s" % request.proxy_auth_mode) else: curl.setopt(pycurl.PROXY, "") curl.unsetopt(pycurl.PROXYUSERPWD) if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) else: curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) if request.ca_certs is not None: curl.setopt(pycurl.CAINFO, request.ca_certs) else: # There is no way to restore pycurl.CAINFO to its default value # (Using unsetopt makes it reject all certificates). # I don't see any way to read the default value from python so it # can be restored later. We'll have to just leave CAINFO untouched # if no ca_certs file was specified, and require that if any # request uses a custom ca_certs file, they all must. pass if request.allow_ipv6 is False: # Curl behaves reasonably when DNS resolution gives an ipv6 address # that we can't reach, so allow ipv6 unless the user asks to disable. curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) else: curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) # Set the request method through curl's irritating interface which makes # up names for almost every single method curl_options = { "GET": pycurl.HTTPGET, "POST": pycurl.POST, "PUT": pycurl.UPLOAD, "HEAD": pycurl.NOBODY, } custom_methods = set(["DELETE", "OPTIONS", "PATCH"]) for o in curl_options.values(): curl.setopt(o, False) if request.method in curl_options: curl.unsetopt(pycurl.CUSTOMREQUEST) curl.setopt(curl_options[request.method], True) elif request.allow_nonstandard_methods or request.method in custom_methods: curl.setopt(pycurl.CUSTOMREQUEST, request.method) else: raise KeyError("unknown method " + request.method) body_expected = request.method in ("POST", "PATCH", "PUT") body_present = request.body is not None if not request.allow_nonstandard_methods: # Some HTTP methods nearly always have bodies while others # almost never do. Fail in this case unless the user has # opted out of sanity checks with allow_nonstandard_methods. if (body_expected and not body_present) or (body_present and not body_expected): raise ValueError( "Body must %sbe None for method %s (unless " "allow_nonstandard_methods is true)" % ("not " if body_expected else "", request.method)) if body_expected or body_present: if request.method == "GET": # Even with `allow_nonstandard_methods` we disallow # GET with a body (because libcurl doesn't allow it # unless we use CUSTOMREQUEST). While the spec doesn't # forbid clients from sending a body, it arguably # disallows the server from doing anything with them. raise ValueError("Body must be None for GET request") request_buffer = BytesIO(utf8(request.body or "")) def ioctl(cmd: int) -> None: if cmd == curl.IOCMD_RESTARTREAD: # type: ignore request_buffer.seek(0) curl.setopt(pycurl.READFUNCTION, request_buffer.read) curl.setopt(pycurl.IOCTLFUNCTION, ioctl) if request.method == "POST": curl.setopt(pycurl.POSTFIELDSIZE, len(request.body or "")) else: curl.setopt(pycurl.UPLOAD, True) curl.setopt(pycurl.INFILESIZE, len(request.body or "")) if request.auth_username is not None: assert request.auth_password is not None if request.auth_mode is None or request.auth_mode == "basic": curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) elif request.auth_mode == "digest": curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) else: raise ValueError("Unsupported auth_mode %s" % request.auth_mode) userpwd = httputil.encode_username_password( request.auth_username, request.auth_password) curl.setopt(pycurl.USERPWD, userpwd) curl_log.debug( "%s %s (username: %r)", request.method, request.url, request.auth_username, ) else: curl.unsetopt(pycurl.USERPWD) curl_log.debug("%s %s", request.method, request.url) if request.client_cert is not None: curl.setopt(pycurl.SSLCERT, request.client_cert) if request.client_key is not None: curl.setopt(pycurl.SSLKEY, request.client_key) if request.ssl_options is not None: raise ValueError("ssl_options not supported in curl_httpclient") if threading.active_count() > 1: # libcurl/pycurl is not thread-safe by default. When multiple threads # are used, signals should be disabled. This has the side effect # of disabling DNS timeouts in some environments (when libcurl is # not linked against ares), so we don't do it when there is only one # thread. Applications that use many short-lived threads may need # to set NOSIGNAL manually in a prepare_curl_callback since # there may not be any other threads running at the time we call # threading.activeCount. curl.setopt(pycurl.NOSIGNAL, 1) if request.prepare_curl_callback is not None: request.prepare_curl_callback(curl)
def test_neither_set(self): proxy = _RequestProxy(HTTPRequest('http://example.com/'), dict()) self.assertIs(proxy.auth_username, None)
async def request(self, method, url, data=None, params=dict()) -> object: client = AsyncHTTPClient() done = False retries = 0 if data is not None: data = json_encode(data) try: def handle_response(response): if 'status' in response and \ 'code' in response['status'] and \ 200 <= int(response['status']['code']) <= 299: return response else: error = response['status'].get('err', {}) or \ response['status'].get('error', {}) or \ response['status'].get('message', {}) or \ url + " request failed with " + response['status'].get('code', 400) + " response" raise APIException(error) url = url_concat(url, params) headers = {'Content-Type': 'application/json'} req = HTTPRequest(url=url, headers=headers, method=method, body=data) async def fetch(): nonlocal retries, req, done retries += 1 with async_timeout.timeout(120): resp = await client.fetch(req) if resp.headers['Content-Type'] == 'application/json': body = json_decode(resp.body) else: body = resp.body done = True return handle_response(body) return await fetch() except asyncio.TimeoutError as ex: if retries <= MAX_RETRY: return await fetch() else: traceback.print_exc() raise ex except HTTPError as ex: if retries <= 3: return await fetch() else: traceback.print_exc() raise ex except Exception as ex: traceback.print_exc() raise ex finally: if done: client.close()
def test_null_headers_setter(self): request = HTTPRequest('http://example.com') request.headers = None self.assertEqual(request.headers, {})
def authenticate(self, handler, data=None): code = handler.get_argument("code") # TODO: Configure the curl_httpclient for tornado http_client = AsyncHTTPClient() params = dict(redirect_uri=self.get_callback_url(handler), code=code, grant_type='authorization_code') if self.token_url: url = self.token_url else: raise ValueError( "Please set the OAUTH2_TOKEN_URL environment variable") b64key = base64.b64encode( bytes("{}:{}".format(self.client_id, self.client_secret), "utf8")) headers = { "Accept": "application/json", "User-Agent": "JupyterHub", "Authorization": "Basic {}".format(b64key.decode("utf8")) } req = HTTPRequest( url, method="POST", headers=headers, body=urllib.parse.urlencode( params) # Body is required for a POST... ) resp = yield http_client.fetch(req) resp_json = json.loads(resp.body.decode('utf8', 'replace')) access_token = resp_json['access_token'] refresh_token = resp_json.get('refresh_token', None) token_type = resp_json['token_type'] scope = (resp_json.get('scope', '')).split(' ') # Determine who the logged in user is headers = { "Accept": "application/json", "User-Agent": "JupyterHub", "Authorization": "{} {}".format(token_type, access_token) } if self.userdata_url: url = url_concat(self.userdata_url, self.userdata_params) else: raise ValueError( "Please set the OAUTH2_USERDATA_URL environment variable") req = HTTPRequest( url, method=self.userdata_method, headers=headers, ) resp = yield http_client.fetch(req) resp_json = json.loads(resp.body.decode('utf8', 'replace')) username = resp_json.get(self.username_key) if not username: username = resp_json.get("username") if not username: self.log.error("OAuth user contains no key %s: %s", self.username_key, resp_json) return if self.authorized_groups: authorized = False if username in self.authorized_groups: authorized = True if not authorized: _group_urls = self.group_urls.split(",") for g_url in _group_urls: groups_req = HTTPRequest(g_url.strip(), method="GET", headers=headers) groups_resp = None try: groups_resp = yield http_client.fetch(groups_req) except HTTPClientError as e: if e.response: self.log.error( "failed to fetch groups for: %s. Reason: %s", g_url, e.response.reason) continue groups_resp_json = json.loads( groups_resp.body.decode('utf8', 'replace')) # Determine whether the user is member of one of the authorized groups user_group_id = [g["id"] for g in groups_resp_json] for group_id in self.authorized_groups.split(","): if group_id in user_group_id: authorized = True break if authorized: break if not authorized: return return { 'name': username, 'auth_state': { 'access_token': access_token, 'refresh_token': refresh_token, 'oauth_user': resp_json, 'scope': scope, } }
def checkresults(db, api): ini = time() info( '=========================================================================' ) info(' Starting CV Request processing - {}'.format( datetime.now().isoformat())) info( '=========================================================================' ) try: time_limit = int(environ.get('CVREQ_TIMELIMIT', 7200)) except Exception as e: info(e) time_limit = 7200 # Check results execute GET in the CV Server URL to acquire results for cv requests AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") http_client = AsyncHTTPClient() cvreqs = db.cvrequests.find() # retrieve data from the cursor lcvreqids = [x['iid'] for x in cvreqs] rmcv = db.cvresults.remove({'cvrequest_iid': { '$nin': lcvreqids }}, multi=True) info(' Clear cvresults without cvrequests: {}'.format(rmcv)) # Get ids with status != finished or error cvreqs = db.cvrequests.find({'status': {'$nin': ['finished', 'error']}}) # retrieve data from the cursor cvreqs = [x for x in cvreqs] info(' CV Request not finished of error - count: ' + str(len(cvreqs))) # Connection preset params = { 'headers': HTTPHeaders({ "content-type": "application/json", "ApiKey": api['CV_APIKEY'] }), 'url': api['CVSERVER_URL'] + '/linc/v1/classify', 'method': 'POST', 'body': '', 'request_timeout': 5, 'validate_cert': False } # Check if cvresults exists for cvreq in cvreqs: info( "========================================================================" ) info(" ### Checking CV Request: " + str(cvreq['iid']) + " ###") info(" ## Image set submitted: " + str(cvreq['image_set_iid']) + " ##") cvres = db.cvresults.find_one({'cvrequest_iid': cvreq['iid']}) # Restart after 10 minutes if cvres: info(' >> Created at: {}'.format(cvres['created_at'])) info(' >> now: {}'.format(datetime.now())) if (datetime.now() - cvres['created_at']).seconds > time_limit: #info(" !!! The recognition process took more than 10 minutes... restarting") info("!!! The CV Request took more than 2 hours to finish") info("!!! Marking it with error status") db.cvrequests.update({'iid': cvreq['iid']}, { '$set': { 'status': 'error', 'updated_at': datetime.now() } }) cvrem_restart = db.cvresults.remove( {'cvrequest_iid': cvreq['iid']}) cvres = None info( "========================================================================" ) continue if not cvres: # Create the CVResults iid = db.counters.find_and_modify(query={'_id': 'cvresults'}, update={'$inc': { 'next': 1 }}, new=True, upsert=True) newcvres = dict() newcvres['cvrequest_iid'] = cvreq['iid'] newcvres['iid'] = iid['next'] newcvres['match_probability'] = '{}' dt = datetime.now() newcvres['created_at'] = dt newcvres['updated_at'] = dt ncvresobjid = db.cvresults.insert(newcvres) info('CV results created id: ' + str(ncvresobjid)) cvres = db.cvresults.find_one({'cvrequest_iid': cvreq['iid']}) # Cvres exists, so try to get data info(' ## CV Results id.....: {} ## '.format(cvres['iid'])) req_body = loads(cvreq['request_body']) resp_cvr = loads(cvres['match_probability']) if len(resp_cvr) == 0: resp_cvr['cv'] = list() resp_cvr['whisker'] = list() # Check for cv results # cv_topk_classifier_accuracy # whisker_topk_classifier_accuracy if not req_body.get('classifiers', False): info(' >>> CV Request invalid - id: {}'.format(cvreq['iid'])) info(' >>> No classifiers found.') else: # Check CV finished = {'cv': 0, 'whisker': 0} for clf in ['cv', 'whisker']: if req_body['classifiers'].get(clf, False): info(' Processing calls for the classifier {}'.format( clf.upper())) add = len(resp_cvr[clf]) == 0 if add: # Submit requests for n, clf_call in enumerate(req_body[clf + '_calls']): dparams = params.copy() dparams['body'] = dumps(clf_call) request = HTTPRequest(**dparams) try: response = yield http_client.fetch(request) except HTTPError as e: info(e) response = e.response except Exception as e: info(e) response = None if response and response.code in [200, 201]: info(' Call {} #{} - success'.format( clf.upper(), n)) resp_cvr[clf].append( loads(response.body.decode('utf-8'))) else: info(' Call {} #{} - fail'.format( clf.upper(), n)) resp_cvr[clf].append('FAILURE') else: # Check results for n, clf_call in enumerate(req_body[clf + '_calls']): info(resp_cvr[clf][n]) # {'id': '432f7612-8b7d-4132-baae-f93f094abb7f', 'status': 'PENDING', 'errors': []} if isinstance(resp_cvr[clf][n], dict) and resp_cvr[clf][n].get( 'status', None) == 'finished': info( ' Request CV #{} finished'.format( n)) finished[clf] += 1 else: info(' Check results for CV #{}'.format( n)) dparams = params.copy() del dparams['body'] dparams['method'] = 'GET' url = api[ 'CVSERVER_URL'] + '/linc/v1/results/' + resp_cvr[ clf][n]['id'] info(' {}'.format(url)) dparams['url'] = url request = HTTPRequest(**dparams) try: response = yield http_client.fetch(request) except HTTPError as e: info(e) response = e.response except Exception as e: info(e) response = None if response.code in [200, 201]: info(' Call #{} - success'.format( n)) resp_data = loads( response.body.decode('utf-8')) info(' Status: {}'.format( resp_data['status'])) if resp_data['status'] == 'finished': info(' Resp data: {}'.format( resp_data)) resp_cvr[clf][n] = resp_data.copy() elif resp_data['status'] == 'error': info( ' Forcing it to be FINISHED (Workaround)' ) finished[clf] += 1 else: info(' Call #{} - fail'.format(n)) dt = datetime.now() if finished['cv'] == len( req_body['cv_calls']) and finished['whisker'] == len( req_body['whisker_calls']): info(' Loading capabilities...') dparams = params.copy() del dparams['body'] dparams['method'] = 'GET' dparams['url'] = api['CVSERVER_URL'] + '/linc/v1/capabilities' request = HTTPRequest(**dparams) try: response = yield http_client.fetch(request) except HTTPError as e: info(e) response = e.response except Exception as e: info(e) response = None if response.code in [200, 201]: info(' ### CV Request finished ###') db.cvrequests.update( {'iid': cvreq['iid']}, {'$set': { 'status': 'finished', 'updated_at': dt }}) resp_cvr['capabilities'] = loads( response.body.decode('utf-8')) resp_cvr['execution'] = dt.timestamp( ) - cvres['created_at'].timestamp() else: info(' Fail to retrieve capabilities info...') db.cvresults.update({'cvrequest_iid': cvreq['iid']}, { '$set': { 'match_probability': dumps(resp_cvr), 'updated_at': dt } }) api['cache'].delete('imgset-' + str(cvreq['image_set_iid'])) info(' Cache delete for image set id: {}'.format( cvreq['image_set_iid'])) info( '=========================================================================' ) info(' CV Request processing finished - Execution time: {0:.2f} s'.format( time() - ini)) info( '=========================================================================' )
def post(self): pay_uid = self.session.get('uid') charge_uid = self.get_argument('charge_uid') goodsId = self.get_argument('goods_id') invite_code = self.get_argument('invite_code', '') res = self.check_invite_code(invite_code, charge_uid) if not res[0]: # self.write("<script>alert('不符合条件,无法充值');window.history.back();</script>") u_error(self, res[1]) raise tornado.gen.Return() out_trade_no, goodsName, goodsPrice, cards, present_card = self.get_trade_and_goods( goodsId, charge_uid) appid = WECHAT_APP_ID mch_id = WECHAT_MCHID nonce_str = "".join( random.sample('zyxwvutsrqponmlkjihgfedcba123456789', 32)) body = "房卡X" + str( int(cards) + int(present_card if present_card else 0)) # 金额 print '----------totalfee---------------' print goodsPrice print '----------totalfee---------------' total_fee = int(float(goodsPrice) * 100) spbill_create_ip = "120.79.229.113" trade_type = "JSAPI" openid = self.session.get("openid") notify_url = "agent.qkgame.com.cn/notify" key = WECHAT_KEY send_list = [ 'appid', 'mch_id', 'nonce_str', 'body', 'out_trade_no', 'total_fee', 'spbill_create_ip', 'trade_type', 'openid', 'notify_url' ] send_list.sort() stringA = "" for val in send_list: stringA += val + "=" + str(eval(val)) + "&" stringA += "key=" + key m = hashlib.md5() m.update(stringA) sign = m.hexdigest().upper() send_list.append('sign') dic = {} for s in send_list: dic[s] = str(eval(s)) dic = {"xml": dic} data = xmltodict.unparse(dic).encode('utf-8') url = "https://api.mch.weixin.qq.com/pay/unifiedorder" client = AsyncHTTPClient() req = HTTPRequest( url, method="POST", body=data, ) # 发送请求获取prepayid resp = yield client.fetch(req) print '--------------' print resp.body print '--------------' res_dic = xmltodict.parse(resp.body) prepay_id = res_dic["xml"]["prepay_id"] timeStamp = int(time.time()) package = "prepay_id=" + str(prepay_id) signType = "MD5" appId = WECHAT_APP_ID nonceStr = nonce_str jssdk_list = ['appId', 'nonceStr', 'timeStamp', 'package', 'signType'] jssdk_list.sort() stringA = "" for val in jssdk_list: stringA += val + "=" + str(eval(val)) + "&" stringA += "key=" + key m = hashlib.md5() m.update(stringA) paySign = m.hexdigest().upper() self.add_order_to_pending(nonceStr, prepay_id, goodsName, total_fee, goodsId, cards, present_card, out_trade_no, paySign, charge_uid) u_success(self, '请求成功', data={ 'signType': signType, 'appId': appId, 'timeStamp': str(timeStamp), 'nonceStr': nonceStr, 'package': package, 'paySign': paySign })
def test_default_set(self): proxy = _RequestProxy(HTTPRequest('http://example.com/'), dict(network_interface='foo')) self.assertEqual(proxy.network_interface, 'foo')
def test_headers(self): request = HTTPRequest('http://example.com', headers={'foo': 'bar'}) self.assertEqual(request.headers, {'foo': 'bar'})
def post(self): uid = self.session.get("uid") money = self.get_argument('money', '') res = self.check_money_valid(money) if not res[0]: u_error(self, res[1]) res = self.check_withdraw_can(uid, money) if not res[0]: u_error(self, res[1]) raise tornado.gen.Return() if self.redis.get('short:forbid:user:%s' % uid): u_error(self, '您的提现过于频繁,请稍等一会!') raise tornado.gen.Return() self.redis.setex('short:forbid:user:%s' % uid, 10, 10) if not money: # self.write("请填写正确的金额!") u_error(self, '请填写正确的金额!') raise tornado.gen.Return() else: try: money_cent = int(float(money) * 100) except Exception as e: # self.write("格式不正确") u_error(self, '格式不正确') raise tornado.gen.Return() if not self.check_have_money(uid, money_cent): # self.write("你的余额不够!") u_error(self, '你的余额不够') raise tornado.gen.Return() openid = self.session.get("openid") mch_appid = WECHAT_APP_ID mchid = WECHAT_MCHID nonce_str = "".join( random.sample('zyxwvutsrqponmlkjihgfedcba123456789', 32)) partner_trade_no = uuid.uuid4().hex check_name = "NO_CHECK" amount = money_cent desc = "代理提现" spbill_create_ip = "120.79.229.113" key = WECHAT_KEY send_list = [ 'openid', 'mch_appid', 'mchid', 'nonce_str', 'partner_trade_no', 'check_name', 'amount', 'desc', 'spbill_create_ip' ] send_list.sort() stringA = "" for val in send_list: stringA += val + "=" + str(eval(val)) + "&" stringA += "key=" + key m = hashlib.md5() m.update(stringA) sign = m.hexdigest().upper() send_list.append('sign') dic = {} for s in send_list: dic[s] = str(eval(s)) dic = {"xml": dic} data = xmltodict.unparse(dic) url = "https://api.mch.weixin.qq.com/mmpaymkttransfers/promotion/transfers" client = AsyncHTTPClient() req = HTTPRequest(url, method="POST", body=data, client_key=self._key_pem, client_cert=self._cert_pem) resp = yield client.fetch(req) res_dic = xmltodict.parse(resp.body) res = self.check_success(res_dic, amount) if res: # self.redirect('/cash/take?withdraw_res=1') u_jump(self, '/cash/take?withdraw_res=1') else: # self.write('提现失败!') try: # res = str(res_dic['xml']['return_msg']) if res_dic['xml']['err_code'] == "NOTENOUGH": res = "商户可用提现余额不足,请不要着急,建议联系客服处理。" elif res_dic['xml']['err_code'] == "SENDNUM_LIMIT": res = "提现次数超出3次,提示:已达到单日最大提现次数,请改天再来。" except Exception as e: print '----except-----------' print str(e) print '---------------' res = "提现失败!" u_error(self, res)
async def authenticate(self, handler, data=None, google_groups=None): code = handler.get_argument("code") body = urllib.parse.urlencode( dict( code=code, redirect_uri=self.get_callback_url(handler), client_id=self.client_id, client_secret=self.client_secret, grant_type="authorization_code", )) req = HTTPRequest( self.token_url, method="POST", headers={"Content-Type": "application/x-www-form-urlencoded"}, body=body, ) user = await self.fetch(req, "completing oauth") access_token = str(user['access_token']) refresh_token = user.get('refresh_token', None) req = HTTPRequest( url_concat( self.user_info_url, {'access_token': access_token}, )) bodyjs = await self.fetch(req, "fetching user info") user_email = bodyjs['email'] user_email_domain = user_email.split('@')[1] username = bodyjs[self.username_claim] if not bodyjs['verified_email']: self.log.warning("Google OAuth unverified email attempt: %s", user_email) raise HTTPError(403, "Google email {} not verified".format(user_email)) if self.hosted_domain: if user_email_domain not in self.hosted_domain: self.log.warning( "Google OAuth unauthorized domain attempt: %s", user_email) raise HTTPError( 403, "Google account domain @{} not authorized.".format( user_email_domain), ) if len(self.hosted_domain) == 1 and user_email == username: # unambiguous domain, use only base name username = user_email.split('@')[0] if refresh_token is None: self.log.debug( "Refresh token was empty, will try to pull refresh_token from previous auth_state" ) user = handler.find_user(username) if user and user.encrypted_auth_state: self.log.debug( "encrypted_auth_state was found, will try to decrypt and pull refresh_token from it" ) try: encrypted = user.encrypted_auth_state auth_state = await decrypt(encrypted) refresh_token = auth_state.get('refresh_token') except (ValueError, InvalidToken, EncryptionUnavailable) as e: self.log.warning( "Failed to retrieve encrypted auth_state for %s because %s", username, e, ) user_info = { 'name': username, 'auth_state': { 'access_token': access_token, 'refresh_token': refresh_token, 'google_user': bodyjs, }, } if self.admin_google_groups or self.allowed_google_groups: user_info = await self._add_google_groups_info( user_info, google_groups) return user_info
def test_if_modified_since(self): http_date = datetime.datetime.utcnow() request = HTTPRequest('http://example.com', if_modified_since=http_date) self.assertEqual(request.headers, {'If-Modified-Since': format_timestamp(http_date)})
def handle_server(user, server_name, server): """Handle (maybe) culling a single server Returns True if server is now stopped (user removable), False otherwise. """ log_name = user['name'] if server_name: log_name = '%s/%s' % (user['name'], server_name) if server.get('pending'): app_log.warning("Not culling server %s with pending %s", log_name, server['pending']) return False if server.get('started'): age = now - parse_date(server['started']) else: # started may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if server['last_activity']: inactive = now - parse_date(server['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'started' field which is never None # for running servers inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info("Culling server %s (inactive for %s)", log_name, format_td(inactive)) if max_age and not should_cull: # only check started if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info("Culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug("Not culling server %s (age: %s, inactive for %s)", log_name, format_td(age), format_td(inactive)) return False body = None if server_name: # culling a named server # A named server can be stopped and kept available to the user # for starting again or stopped and removed. To remove the named # server we have to pass an additional option in the body of our # DELETE request. delete_url = url + "/users/%s/servers/%s" % ( quote(user['name']), quote(server['name']), ) if remove_named_servers: body = json.dumps({"remove": True}) else: delete_url = url + '/users/%s/server' % quote(user['name']) req = HTTPRequest( url=delete_url, method='DELETE', headers=auth_header, body=body, allow_nonstandard_methods=True, ) resp = yield fetch(req) if resp.code == 202: app_log.warning( "Server %s is slow to stop", log_name, ) # return False to prevent culling user with pending shutdowns return False return True
def test_request_set(self): proxy = _RequestProxy( HTTPRequest('http://example.com/', user_agent='foo'), dict()) self.assertEqual(proxy.user_agent, 'foo')
async def test_client_fetch_timeout(): async with slow_server() as server: gateway = Gateway(server.address, auth=BasicAuth("alice")) with pytest.raises(TimeoutError): await gateway._fetch( HTTPRequest(url=server.address, request_timeout=1))
def authenticate(self, handler, data=None): """We set up auth_state based on additional CILogon info if we receive it. """ code = handler.get_argument("code") # TODO: Configure the curl_httpclient for tornado http_client = AsyncHTTPClient() # Exchange the OAuth code for a CILogon Access Token # See: http://www.cilogon.org/oidc headers = { "Accept": "application/json", "User-Agent": "JupyterHub", } params = dict( client_id=self.client_id, client_secret=self.client_secret, redirect_uri=self.oauth_callback_url, code=code, grant_type='authorization_code', ) url = url_concat("https://%s/oauth2/token" % CILOGON_HOST, params) req = HTTPRequest(url, headers=headers, method="POST", body='') resp = yield http_client.fetch(req) token_response = json.loads(resp.body.decode('utf8', 'replace')) access_token = token_response['access_token'] self.log.info("Access token acquired.") # Determine who the logged in user is params = dict(access_token=access_token) req = HTTPRequest(url_concat( "https://%s/oauth2/userinfo" % CILOGON_HOST, params), headers=headers) resp = yield http_client.fetch(req) resp_json = json.loads(resp.body.decode('utf8', 'replace')) claimlist = [self.username_claim] if self.additional_username_claims: claimlist.extend(self.additional_username_claims) for claim in claimlist: username = resp_json.get(claim) if username: break if not username: if len(claimlist) < 2: self.log.error("Username claim %s not found in response: %s", self.username_claim, sorted(resp_json.keys())) else: self.log.error("No username claim from %r in response: %s", claimlist, sorted(resp_json.keys())) raise web.HTTPError(500, "Failed to get username from CILogon") if self.idp_whitelist: gotten_name, gotten_idp = username.split('@') if gotten_idp not in self.idp_whitelist: self.log.error( "Trying to login from not whitelisted domain %s", gotten_idp) raise web.HTTPError( 500, "Trying to login from not whitelisted domain") if len(self.idp_whitelist) == 1 and self.strip_idp_domain: username = gotten_name userdict = {"name": username} # Now we set up auth_state userdict["auth_state"] = auth_state = {} # Save the token response and full CILogon reply in auth state # These can be used for user provisioning # in the Lab/Notebook environment. auth_state['token_response'] = token_response # store the whole user model in auth_state.cilogon_user # keep access_token as well, in case anyone was relying on it auth_state['access_token'] = access_token auth_state['cilogon_user'] = resp_json return userdict
def request(self, url, data=None, method="GET", tried=False, full_response=False): #print("Token: {0}".format(self.token_manager.token)) if self.token_manager.token in [None, ""]: yield self.get_token() tried = True headers = self.get_headers() #print("Headers:{0}".format(headers)) if data != None: if method in [None, "GET"]: method = "POST" print("building request: {0}, {1}, {2}".format(method, url, data)) request = HTTPRequest(url, method=method, headers=headers, body=data) else: request = HTTPRequest(url, method=method, headers=headers) http_client = AsyncHTTPClient() try: response = yield http_client.fetch(request) if not full_response: message = json.loads(response.body.decode("utf-8")) response = { "success": True, "code": response.code, "data": message, "headers": response.headers._dict } except HTTPError as he: print("Requester.request HTTPError Code: {0}, {1}".format( he.code, he.message)) try: print(he.response.body) except Exception as e: pass if not full_response: try: response = { "success": False, "code": he.code, "message": he.message, "headers": he.response.headers._dict } except Exception as ex: response = { "success": False, "code": he.code, "message": he.message } else: response = he.response if not tried: if he.code == 401: print("Token may have expired, regenerating") yield self.get_token() response = yield self.request(url, data, method, True, full_response) elif he.code == 400: try: print("request error he.response.body:{0}".format( he.response.body)) jval = json.loads(he.response.body.decode('utf-8')) if "Invalid access token" in jval['error']['message'][ 'error_description']: yield self.get_token() response = yield self.request( url, data, method, True, full_response) except Exception as ex: print(ex) elif he.code == 429: print("SHOULD DO SOMETHING HERE FOR 429") else: print("SOME OTHER CODE: {0}".format(he.code)) #response = yield self.request(url, data, method, True, full_response) else: print("Failed on second attempt.") except Exception as e: traceback.print_exc() message = "{0}".format(e) if not full_response: response = { "success": False, "code": 500, "message": message, "headers": {} } else: response = None raise tornado.gen.Return(response)
def github_api_request(self, api_url, etag=None): client = AsyncHTTPClient() if self.auth: # Add auth params. After logging! api_url = url_concat(api_url, self.auth) headers = {} if etag: headers['If-None-Match'] = etag req = HTTPRequest(api_url, headers=headers, user_agent="BinderHub") try: resp = yield client.fetch(req) except HTTPError as e: if e.code == 304: resp = e.response elif ( e.code == 403 and e.response and e.response.headers.get('x-ratelimit-remaining') == '0' ): rate_limit = e.response.headers['x-ratelimit-limit'] reset_timestamp = int(e.response.headers['x-ratelimit-reset']) reset_seconds = int(reset_timestamp - time.time()) self.log.error( "GitHub Rate limit ({limit}) exceeded. Reset in {delta}.".format( limit=rate_limit, delta=timedelta(seconds=reset_seconds), ) ) # round expiry up to nearest 5 minutes minutes_until_reset = 5 * (1 + (reset_seconds // 60 // 5)) raise ValueError("GitHub rate limit exceeded. Try again in %i minutes." % minutes_until_reset ) # Status 422 is returned by the API when we try and resolve a non # existent reference elif e.code in (404, 422): return None else: raise # record and log github rate limit remaining = int(resp.headers['x-ratelimit-remaining']) rate_limit = int(resp.headers['x-ratelimit-limit']) reset_timestamp = int(resp.headers['x-ratelimit-reset']) # record with prometheus GITHUB_RATE_LIMIT.set(remaining) # log at different levels, depending on remaining fraction fraction = remaining / rate_limit if fraction < 0.2: log = self.log.warning elif fraction < 0.5: log = self.log.info else: log = self.log.debug # str(timedelta) looks like '00:32' delta = timedelta(seconds=int(reset_timestamp - time.time())) log("GitHub rate limit remaining {remaining}/{limit}. Reset in {delta}.".format( remaining=remaining, limit=rate_limit, delta=delta, )) return resp
def test_bad_attribute(self): proxy = _RequestProxy(HTTPRequest('http://example.com/'), dict()) with self.assertRaises(AttributeError): proxy.foo
def test_headers_setter(self): request = HTTPRequest('http://example.com') request.headers = {'bar': 'baz'} self.assertEqual(request.headers, {'bar': 'baz'})
def test_both_set(self): proxy = _RequestProxy( HTTPRequest('http://example.com/', proxy_host='foo'), dict(proxy_host='bar')) self.assertEqual(proxy.proxy_host, 'foo')
def test_body_setter(self): request = HTTPRequest('http://example.com') request.body = 'foo' self.assertEqual(request.body, utf8('foo'))
async def ws_client(http_server, http_server_port): """ return a websocket client """ request = HTTPRequest(f'ws://localhost:{http_server_port[1]}/ws') result = await websocket_connect(request) return result
async def authenticate(self, handler, data=None): code = handler.get_argument("code") # Exchange the OAuth code for a GitLab Access Token # # See: https://github.com/gitlabhq/gitlabhq/blob/HEAD/doc/api/oauth2.md # GitLab specifies a POST request yet requires URL parameters params = dict( client_id=self.client_id, client_secret=self.client_secret, code=code, grant_type="authorization_code", redirect_uri=self.get_callback_url(handler), ) validate_server_cert = self.validate_server_cert url = url_concat("%s/oauth/token" % self.gitlab_url, params) req = HTTPRequest( url, method="POST", headers={"Accept": "application/json"}, validate_cert=validate_server_cert, body='', # Body is required for a POST... ) resp_json = await self.fetch(req, label="getting access token") access_token = resp_json['access_token'] # memoize gitlab version for class lifetime if self.gitlab_version is None: self.gitlab_version = await self._get_gitlab_version(access_token) self.member_api_variant = 'all/' if self.gitlab_version >= [12, 4] else '' # Determine who the logged in user is req = HTTPRequest( "%s/user" % self.gitlab_api, method="GET", validate_cert=validate_server_cert, headers=_api_headers(access_token), ) resp_json = await self.fetch(req, label="getting gitlab user") username = resp_json["username"] user_id = resp_json["id"] is_admin = resp_json.get("is_admin", False) # Check if user is a member of any allowed groups or projects. # These checks are performed here, as it requires `access_token`. user_in_group = user_in_project = False is_group_specified = is_project_id_specified = False if self.allowed_gitlab_groups: is_group_specified = True user_in_group = await self._check_membership_allowed_groups( user_id, access_token ) # We skip project_id check if user is in allowed group. if self.allowed_project_ids and not user_in_group: is_project_id_specified = True user_in_project = await self._check_membership_allowed_project_ids( user_id, access_token ) no_config_specified = not (is_group_specified or is_project_id_specified) if ( (is_group_specified and user_in_group) or (is_project_id_specified and user_in_project) or no_config_specified ): return { 'name': username, 'auth_state': {'access_token': access_token, 'gitlab_user': resp_json}, } else: self.log.warning("%s not in group or project allowed list", username) return None