def post_suggest(self, user_id: str, application_id: str, session_id: str, locale: str, context: dict, callback) -> str: self.logger.debug( "user_id=%s,application_id=%s,session_id=%s,locale=%s," "context=%s", user_id, application_id, session_id, locale, context ) url = "%s?session_id=%s&application_id=%s&locale=%s" % ( SUGGEST_URL, session_id, application_id, locale ) url += "&user_id=%s" % user_id if user_id is not None else "" try: request_body = { "context": context } http_client = AsyncHTTPClient() http_client.fetch(HTTPRequest(url=url, body=dumps(request_body), method="POST"), callback=callback) http_client.close() except HTTPError: self.logger.error("url=%s", url) raise
def post_context_message( self, context_id: str, direction: int, message_text: str, callback, detection: dict = None, now=None): """ Direction is 1 user 0 jemboo :type direction: int """ self.logger.debug( "context_id=%s,direction=%s,message_text=%s,detection=%s", context_id, direction, message_text, detection ) now = datetime.now() if now is None else now try: request_body = { "direction": direction, "text": message_text, "created": now.isoformat() } if detection is not None: request_body["detection"] = detection url = "%s/%s/messages/" % (CONTEXT_URL, context_id) http_client = AsyncHTTPClient() http_client.fetch( HTTPRequest(url=url, method="POST", body=json_encode(request_body)), callback=callback ) http_client.close() except HTTPError as e: self.logger.error("post_context_message,url=%s", url) raise
def _check_heapster_status(settings): if "HEAPSTER_SERVICE_HOST" in settings: heapster_endpoint = settings["HEAPSTER_SERVICE_HOST"] heapster_port = settings["HEAPSTER_SERVICE_PORT"] endpoint = "http://%s:%s/api/v1/model" % (heapster_endpoint, heapster_port) headers = dict() else: kubernetes_endpoint = settings["KUBERNETES_SERVICE_HOST"] if not kubernetes_endpoint.startswith("http"): kubernetes_endpoint = "https://%s:%s" % (kubernetes_endpoint, settings["KUBERNETES_SERVICE_PORT"]) endpoint = "%s%s" % ( kubernetes_endpoint, "/api/v1/proxy/namespaces/kube-system/services/heapster/api/v1/model" ) headers = dict(Authorization="Bearer %s" % settings['token']) client = AsyncHTTPClient(force_instance=True) try: result = yield client.fetch( endpoint + "/metrics", method="GET", validate_cert=False, headers=headers, raise_error=False) if not result.error: raise Return(status_ok()) else: raise Return(status_error(result.error)) finally: client.close()
def post(self, *args, **kwargs): # TODO # save the messages to local database # file = self.request.files['image'][0] # file_name = file["filename"] # image = file['body'] text = self.get_argument("text") data = { "from": "Mailgun Sandbox <*****@*****.**>", "to": "<*****@*****.**>", "subject": "Hello Udaan", "text": text, } data = urlencode(data) client = AsyncHTTPClient() headers_object = HTTPHeaders({"X-Mailgun-Variables": dumps({"X-Mailgun-Variables": {"password": "******"}})}) request_object = HTTPRequest("https://api.mailgun.net/v3/sandbox1713f24a60034b5ab5e7fa0ca2faa9b6.mailgun.org" "/messages", method="POST", headers=headers_object, body=data, auth_username="******", auth_password="******" ) print(request_object.headers.get_list("X-Mailgun-Variables")) response = yield client.fetch(request_object) client.close() print(response) if response.code == 200: msg = "email send successfully" self.respond(msg, response.code) else: msg = "Please try again" self.respond(msg, response.code)
def request(self, url, method="GET", **kwargs): client = AsyncHTTPClient(force_instance=True) try: result = yield client.fetch(url, method=method, headers=self.build_headers(), **kwargs) raise Return(result) finally: client.close()
class Worker(object): # Seconds before retrying a request to server in seconds POLL_INTERVAL = 1 # Seconds to wait when there are no new jobs NO_JOB_WAIT= 60 def __init__(self, api_url, mongo_uri, **kwargs): self.url = api_url self.name = socket.gethostname() self.http_client = AsyncHTTPClient() self.models = {"keras": KerasModel(**kwargs)} self.data_path = os.path.join(os.path.dirname(__file__), "data") self.data_loader = DataLoader(api_url, mongo_uri) def json_decode(self, body): json_body = json.loads(body.decode("utf-8")) # This needs to be called two times on escaped json string. ¯\_(ツ)_/¯ if not isinstance(json_body, dict): return json.loads(json_body) return json_body @gen.coroutine def get_new_job(self): try: response = yield self.http_client.fetch(self.url + "/worker/" + self.name + "/job") job = self.json_decode(response.body) # no jobs, wait 1 min before next request. if "wait" in job: print "Experiment %s is done! There are no new jobs yet." % job["experiment_id"] IOLoop.current().call_later(self.NO_JOB_WAIT, lambda: self.get_new_job()) return # if job type is unsupported (only keras supported at the moment) if job["model_type"] not in self.models: raise ValueError("Only the following models are supported right now: " + ", ".join(self.models.keys())) # TODO: clean it up! terrible, I know.. model = self.models[job["model_type"]] model.exp_id = job["experiment_id"] job["model_params"]["data_filename"] = job["data_filename"] result = model.run_job(job["model_params"]) # Get a new job from server IOLoop.current().call_later(1, lambda: self.get_new_job()) except HTTPError as e: print("Error: " + str(e)) IOLoop.current().call_later(self.POLL_INTERVAL, lambda: self.get_new_job()) except Exception as e: # Other errors are possible, such as IOError. print("Error: " + str(e)) IOLoop.current().call_later(self.POLL_INTERVAL, lambda: self.get_new_job()) def close(self): self.http_client.close()
class AsyncHTTPTestCase(AsyncTestCase): '''A test case that starts up an HTTP server. Subclasses must override get_app(), which returns the tornado.web.Application (or other HTTPServer callback) to be tested. Tests will typically use the provided self.http_client to fetch URLs from this server. Example: class MyHTTPTest(AsyncHTTPTestCase): def get_app(self): return Application([('/', MyHandler)...]) def test_homepage(self): self.http_client.fetch(self.get_url('/'), self.stop) response = self.wait() # test contents of response ''' def setUp(self): super(AsyncHTTPTestCase, self).setUp() self.__port = None self.http_client = AsyncHTTPClient(io_loop=self.io_loop) self._app = self.get_app() self.http_server = HTTPServer(self._app, io_loop=self.io_loop, **self.get_httpserver_options()) self.http_server.listen(self.get_http_port()) def get_app(self): """Should be overridden by subclasses to return a tornado.web.Application or other HTTPServer callback. """ raise NotImplementedError() def get_httpserver_options(self): """May be overridden by subclasses to return additional keyword arguments for HTTPServer. """ return {} def get_http_port(self): """Returns the port used by the HTTPServer. A new port is chosen for each test. """ if self.__port is None: self.__port = get_unused_port() return self.__port def get_url(self, path): """Returns an absolute url for the given path on the test server.""" return 'http://localhost:%s%s' % (self.get_http_port(), path) def tearDown(self): self.http_server.stop() self.http_client.close() super(AsyncHTTPTestCase, self).tearDown()
def get(self, url_path, **kwargs): params = self.build_params(url_path, **kwargs) url = url_concat(self.build_url(url_path, **kwargs), params) client = AsyncHTTPClient(force_instance=True) try: result = yield client.fetch(url, method="GET", headers=self.build_headers()) raise Return(result) finally: client.close()
def get_detect(self, location: str, callback) -> dict: self.logger.debug("location=%s", location) url = "%s%s" % (DETECT_URL, location) try: http_client = AsyncHTTPClient() http_client.fetch(HTTPRequest(url=url, method="GET"), callback=callback) http_client.close() except HTTPError as e: self.logger.error("get_detect,url=%s", url) raise
def delete(self, url_path, **kwargs): client = AsyncHTTPClient(force_instance=True) try: response = yield client.fetch( self.build_url(url_path, **kwargs), method="DELETE", headers=self.build_headers()) raise Return(response) finally: client.close()
def get_context_messages(self, handler: WebSocketHandler, callback) -> dict: try: if handler.context is None or handler.context["_rev"] != handler.context_rev: self.logger.debug( "get_context_from_service,context_id=%s,_rev=%s", str(handler.context_id), handler.context_rev) http_client = AsyncHTTPClient() url = "%s/%s/messages" % (CONTEXT_URL, str(handler.context_id)) http_client.fetch(HTTPRequest(url=url, method="GET"), callback=callback) http_client.close() except HTTPError as e: self.logger.error("get_context,url=%s", url) raise
def async_fetch(self, url, method=HTTP_METHOD_GET, params=None, callback=None): """ """ if not callback: raise HttpClientCallbackError, "callback is wrong" # 先打包数据 request = self._pack_data(url, method, params) # 发起请求 http_client = AsyncHTTPClient() http_client.fetch(request, callback=callback) http_client.close()
def coroutine_fetch(self, url, method=HTTP_METHOD_GET, params=None): """ """ # 打包数据 request = self._pack_data(url, method, params) http_client = AsyncHTTPClient() response = yield http_client.fetch(request) http_client.close() if not response: yield response return yield response.body
def request(self, url_path, method="GET", **kwargs): params = self.build_params(url_path, **kwargs) if not urlparse.urlparse(url_path).netloc: url = url_concat(self.build_url(url_path, **kwargs), params) else: url = url_concat(url_path, params) client = AsyncHTTPClient(force_instance=True) try: result = yield client.fetch(url, method=method, headers=self.build_headers()) raise Return(result) finally: client.close()
def run_client(): client = AsyncHTTPClient() range_it = xrange(loops) t0 = perf.perf_counter() for _ in range_it: futures = [client.fetch(url) for j in xrange(CONCURRENCY)] for fut in futures: resp = yield fut buf = resp.buffer buf.seek(0, 2) assert buf.tell() == len(CHUNK) * NCHUNKS namespace['dt'] = perf.perf_counter() - t0 client.close()
def patch(self, url_path, **kwargs): url = self.build_url(url_path, **kwargs) params = self.build_params(url_path, **kwargs) client = AsyncHTTPClient(force_instance=True) try: result = yield client.fetch( url, method="PATCH", headers=self.build_headers("application/merge-patch+json"), **params) raise Return(result) finally: client.close()
def __call__(self, message): http_client = AsyncHTTPClient(force_instance=self._force_instance) debug('client = %s, max_buffer_size = %s', http_client.__class__.__name__, getattr(http_client, 'max_buffer_size') if hasattr(http_client, 'max_buffer_size') else 'unsupported') response = ('', None) try: response = yield http_client.fetch(self.url, body=message, method='POST', connect_timeout=self.connection_timeout, request_timeout=self.connection_timeout) except HTTPError as e: error("HTTP Error: %s, payload size (bytes): %s", e, len(message)) handle_exception(ErrorMessage.from_exception(e, address=self.url)) finally: if self._force_instance: http_client.close() raise gen.Return(response)
def send_async_request(url, headers=None, method="GET", body=None): headers = headers or {} if body or method.upper() == "POST": if 'Content-Type' not in headers: headers["Content-Type"] = "application/x-www-form-urlencoded" req = httpclient.HTTPRequest( url, method=method, body=body, headers=headers, allow_nonstandard_methods=True) http_request = AsyncHTTPClient() response = "" try: response = yield http_request.fetch(req) except httpclient.HTTPError as e: print("Error:" + str(e)) except Exception as e: print("Error:" + str(e)) else: return response finally: http_request.close()
class AsyncHTTPTestCase(AsyncTestCase): def setup(self): self.stopped = False self.running = False self.failure = None self.stop_args = None self.http_client = AsyncHTTPClient(io_loop=self.io_loop) if 'get_app' in dir(self.__class__): # the app needs its own io loop self.io_loop = self.get_new_ioloop() self.port = get_unused_port() self.app = self.get_app() self.http_server = HTTPServer(self.app, io_loop=self.io_loop, **self.get_httpserver_options()) self.http_server.listen(self.port) else: # if we don't test handlers use the clients io loop self.io_loop = self.http_client.io_loop def fetch(self, path, **kwargs): """ Simple wrapper around ``http_client``. If the given ``path`` doesn't start with 'http' than ``path`` is passed on to the ``http_client.fetch`` """ self.http_client.fetch(self.get_url(path), self.stop, **kwargs) return self.wait() def get_httpserver_options(self): return {} def get_url(self, path): if not path.startswith('http'): return 'http://localhost:%s%s' % (self.port, path) return path def teardown(self): if 'http_server' in dir(self.__class__): self.http_server.stop() if 'http_client' in dir(self.__class__): self.http_client.close()
def run_client(): client = AsyncHTTPClient() range_it = xrange(loops) if loops == -1: # -1 is forever range_it = iter(int, 1) else: range_it = xrange(loops) for _ in range_it: futures = [client.fetch(url) for j in xrange(CONCURRENCY)] for fut in futures: resp = yield fut buf = resp.buffer buf.seek(0, 2) assert buf.tell() == len(CHUNK) * NCHUNKS time.sleep(sleep) client.close()
def xicheng_callback(order_id, result): desc = '' if ';' in result: result, desc = result.split(';') # cust_id + echo + secret + tsp sign = signature64('njqxwlkj' + '2016011422063442' + 'njqxwl@QX' + '20160114220634') body = BODY.format(sign=sign, order_id=order_id, result=result, desc=desc) http_client = AsyncHTTPClient() try: request_log.info('XICHENG CALLBACK\n%s', body) h = {'Content-Type': 'application/xml;charset=UTF-8'} http_client.fetch('http://localhost:8899/callback/xicheng', headers=h, method='POST', body=body) except Exception: request_log.exception('FAIL') finally: http_client.close()
def query_order_by_jobid(handler, partner, jobId): userNo = partner['userNo'] userKey = partner['userKey'] query_body = { "userNo": userNo, "userKey": userKey, "jobId": jobId, } url = partner['url.query2'] request_body = urlencode(query_body) url = url + '?' + request_body http_client = AsyncHTTPClient() try: request_log.info("SUOPAI REQ %s", request_body, extra={'orderid': handler.order_id}) response = yield http_client.fetch(url, method='GET', request_timeout=120) except HTTPError as http_error: request_log.error('SUOPAI CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id}) # result = 60000 + http_error.code response = None except Exception as e: request_log.error('SUOPAI CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id}) response = None finally: http_client.close() return response
def post(self, product): args = self.json_args if 'admin' not in self.current_user['roles'] or 'user_id' not in args: args['user_id'] = self.current_user['partner_id'] if 'blocking' in self.application.config and 'url' in self.application.config[ 'blocking']: http_client = AsyncHTTPClient() try: url = '%s/query/%s/export' % ( self.application.config['blocking']['url'], product) request = HTTPRequest(url=url, method='POST', body=json.dumps(args), request_timeout=120) response = yield http_client.fetch(request) if response.code == 200: body = response.body.decode('utf8') self.finish(body) else: self.send_error(500) except Exception as e: request_log.exception('QUERY CHAIN') self.send_error(500) finally: http_client.close() else: # fallback o = OrderBlockingQueryHandler(self.application, self.request) o.json_args = args o.post(product) self._headers = o._headers self._status_code = o._status_code self._write_buffer = o._write_buffer self.finish()
def get_suggestion_items(self, user_id: str, application_id: str, session_id: str, locale: str, suggestion_id: str, page_size: int, offset: int, callback): self.logger.debug( "user_id=%s,application_id=%s,session_id=%s,locale=%s," "suggestion_id=%s,page_size=%s,offset=%s", user_id, application_id, session_id, locale, suggestion_id, page_size, offset ) url = "%s/%s/items?session_id=%s&application_id=%s&locale=%s&page_size=%s&offset=%s" % ( SUGGEST_URL, suggestion_id, session_id, application_id, locale, page_size, offset ) url += "&user_id=%s" % user_id if user_id is not None else "" try: http_client = AsyncHTTPClient() http_client.fetch(HTTPRequest(url=url, method="GET"), callback=callback) http_client.close() except HTTPError: self.logger.error("url=%s", url) raise
class from_web_stream(Stream): def __init__(self, url='http://127.0.0.1:9999', read_timeout=60 * 60 * 24, start=True, **kwargs): self.url = url self.request_timeout = read_timeout self.http_client = AsyncHTTPClient() super(from_web_stream, self).__init__(ensure_io_loop=True, **kwargs) self.stopped = True if start: self.start() @gen.coroutine def get(self): # if self.read_timeout is None: # HTTPRequest._DEFAULTS['request_timeout']=None # defaultset none is 20s,hack源代码也无法解决的,这部分代码不管用 requests = HTTPRequest(url=self.url, streaming_callback=self.on_chunk, request_timeout=self.request_timeout) yield self.http_client.fetch(requests) @gen.coroutine def on_chunk(self, chunk): chunk >> self def start(self): if self.stopped: self.stopped = False if self.http_client is None: self.http_client = AsyncHTTPClient() self.get() def stop(self): if self.http_client is not None: self.http_client.close() # this not imple self.http_client = None self.stopped = True
def dispatch_mutil_task(request_ip_port_params_list, uri, http_method): http_client = AsyncHTTPClient() _error_record_dict = {} adminUser, adminPasswd = _retrieve_userName_passwd() try: _key_sets = set() for (req_ip, req_port, params) in request_ip_port_params_list: requesturl = "http://%s:%s%s" % (req_ip, req_port, uri) logging.info('requesturi: %s' % requesturl) logging.info('dispatch mutil task params :%s' % str(params)) request = HTTPRequest(url=requesturl, method=http_method, body=urllib.urlencode(params), \ connect_timeout=40, request_timeout=40, auth_username=adminUser, auth_password=adminPasswd) callback_key = "%s_%s_%s" % (uri, req_ip, req_port) _key_sets.add(callback_key) http_client.fetch(request, callback=(yield Callback(callback_key))) for callback_key in _key_sets: response = yield Wait(callback_key) if response.error: return_result = False error_record_msg = "remote access,the key:%s,error message:%s" % ( callback_key, response.error) else: return_result = response.body.strip() if cmp('false', return_result) == 0: callback_key_ip = callback_key.split("_")[-1] _error_record_dict.setdefault(callback_key_ip, error_record_msg) if len(_error_record_dict) > 0: raise CommonException('request occurs error! detail: %s' % str(_error_record_dict)) else: logging.info('request finished all!') finally: http_client.close()
async def get(self, pkg_url): async def download_success(req, file_path: str, content: bytes): async with AIOFile(file_path, 'wb') as afp: await afp.write(content) await afp.fsync() req.write(content) req.set_header('Content-Type', 'application/octet-stream') await req.finish() log.info("fetch package {}".format(pkg_url)) pkg_path = pkg_base_path / pkg_url if not pkg_path.parent.exists(): pkg_path.parent.mkdir(parents=True) http_client = AsyncHTTPClient() pkg_url_custom = None try: results = re.search(re_suffix, pkg_url) if results is None: raise Exception("version not recognised {}".format(pkg_url)) pkg_id = pkg_url.split("/-/")[0] pkg_url_custom = "{}/{}/download/{}{}".format( pkg_base_custom, pkg_id, pkg_id, results.group()) r = await http_client.fetch(pkg_url_custom) except Exception as e: log.warning("fetch {} failed: {}".format(pkg_url_custom, e)) pkg_url_npm = "{}/{}".format(pkg_base_npm, pkg_url) try: r = await http_client.fetch(pkg_url_npm, request_timeout=300.0) except Exception as e: log.error("fetch {} failed: {}".format(pkg_url_npm, e)) else: await download_success(self, str(pkg_path), r.body) else: await download_success(self, str(pkg_path), r.body) finally: http_client.close()
def post(self, product): args = self.json_args user_id = self.current_user['partner_id'] args['user_id'] = user_id args['criteria']['product'] = product if 'admin' not in self.current_user['roles']: args['criteria']['user_id'] = user_id body = json.dumps(args) http_client = AsyncHTTPClient() try: downstream = self.application.config['downstream'][user_id] iv = downstream['iv'] passphrase = downstream['pass'] aes = AES.new(passphrase, AES.MODE_CBC, iv) b = aes.encrypt(padding(body)) encrypted = base64.b64encode(b).decode() base_url = self.application.config['connection']['glados_hall'] url = base_url + '/export/request' headers = {'User': user_id} response = yield http_client.fetch(url, method='POST', headers=headers, body=encrypted) resp = response.body.decode() self.finish(resp) except: request_log.exception('EXPORT FAIL') self.finish(json.dumps({'status': 'fail', 'msg': '导出异常,请稍后尝试'})) finally: http_client.close()
def fetch(self, url, method=None, headers=None, body=None, callback=None): """ 执行请求 :param url: :param method: :param headers: :param body: :param callback: 回调函数的值为HttpResponse :return: """ http_client = AsyncHTTPClient() try: request = HTTPRequest(url, method=method, headers=headers, body=body) http_client.fetch(request, callback) except HTTPError as e: gen_log.warning("HttpHelper fetch Error: " + str(e)) except Exception as e: gen_log("HttpHelper fetch Error: " + str(e)) http_client.close()
def post_detect(self, user_id: str, application_id: str, session_id: str, locale: str, query: str, callback) -> str: self.logger.debug( "user_id=%s,application_id=%s,session_id=%s,locale=%s,query=%s", user_id, application_id, session_id, locale, query ) url = "%s?application_id=%s&session_id=%s&locale=%s&q=%s" % ( DETECT_URL, application_id, session_id, locale, url_escape(query) # url_escape(json_encode(context)) ) if user_id is not None: url += "&user_id=%s" % user_id http_client = AsyncHTTPClient() http_client.fetch( HTTPRequest(url=url, method="POST", body=json_encode({})), callback=callback ) http_client.close()
def delete_favorite(self, handler: WebSocketHandler, user_id: ObjectId, product_id: ObjectId): def delete_favorite_callback(response, handler): self.favorites_cache.remove(user_id) self.logger.debug("delete_favorite_completed") self.logger.debug( "user_id=%s,product_id=%s", user_id, product_id ) try: url = "%s/%s/favorite/%s" % (USER_URL, user_id, product_id) http_client = AsyncHTTPClient() http_client.fetch( HTTPRequest(url=url, method="DELETE"), callback=lambda res: delete_favorite_callback(res, handler) ) http_client.close() except HTTPError as e: self.logger.error("delete_favorite,url=%s", url) raise
def post_update(self): http_client = AsyncHTTPClient() try: base_url = self.application.config['connection']['repo'] url = base_url + '/api/user/update' self.json_args['domain_id'] = self.current_user['domain_id'] body = json.dumps(self.json_args) response = yield http_client.fetch(url, method='POST', body=body) resp = response.body.decode() self.finish(resp) except ValueError as e: status = 'fail' msg = '代理商更新失败:' + str(e) self.finish(json.dumps({'status': status, 'msg': msg})) finally: http_client.close()
def get_suggestion_items(self, user_id: str, application_id: str, session_id: str, locale: str, suggestion_id: str, page_size: int, offset: int, callback): self.logger.debug( "user_id=%s,application_id=%s,session_id=%s,locale=%s," "suggestion_id=%s,page_size=%s,offset=%s", user_id, application_id, session_id, locale, suggestion_id, page_size, offset) url = "%s/%s/items?session_id=%s&application_id=%s&locale=%s&page_size=%s&offset=%s" % ( SUGGEST_URL, suggestion_id, session_id, application_id, locale, page_size, offset) url += "&user_id=%s" % user_id if user_id is not None else "" try: http_client = AsyncHTTPClient() http_client.fetch(HTTPRequest(url=url, method="GET"), callback=callback) http_client.close() except HTTPError: self.logger.error("url=%s", url) raise
def aspire_ec_callback(OperSeq, mobile_list): #body = CALLBACK.format(TransIDO, MobNum) SuccNum = 0 SuccessTelList = '' FailNum = 0 FailTelList = '' # for mobile in mobile_list: # SuccessTelList += SUCCESS_TEL.format(Mobile = mobile) for index, mobile in enumerate(mobile_list): if index % 2 == 0: SuccNum += 1 SuccessTelList += SUCCESS_TEL.format(Mobile=mobile) else: FailNum += 1 FailTelList += FAIL_TEL.format(Mobile=mobile) body = CALLBACK_SUCC.format(OperSeq=OperSeq, SuccNum=SuccNum, SuccessTelList=SuccessTelList, FailNum=FailNum, FailTelList=FailTelList) http_client = AsyncHTTPClient() try: request_log.info('ASPIRE_EC CALLBACK\n%s', body) http_client.fetch( 'http://192.168.1.152:8899/callback/cmcc3.do', method='POST', body=body, headers={'Content-Type': 'application/xml;charset=UTF-8'}) except Exception: request_log.exception('FAIL') finally: http_client.close()
def post_approval(self): http_client = AsyncHTTPClient() try: user_name = self.json_args.get('name') login = self.json_args.get('login') request_id = self.json_args.get('request_id') if user_name is None or user_name == '': raise ValueError('用户名为空') if login is None or login == '': raise ValueError('登录名为空') self.json_args['domain_id'] = self.current_user['domain_id'] body = json.dumps(self.json_args) base_url = self.application.config['connection']['repo'] url = base_url + '/api/user/add' response = yield http_client.fetch(url, method='POST', body=body) resp = response.body.decode() self.finish(resp) if request_id and self.master.sismember('set:user:request', request_id): self.master.srem('set:user:request', request_id) self.master.delete('request:%s' % request_id) except Exception as e: status = 'fail' msg = '代理商添加失败:' + str(e) self.finish(json.dumps({'status': status, 'msg': msg})) finally: http_client.close()
def _check_heapster_available(self): if "HEAPSTER_SERVICE_HOST" in os.environ: heapster_endpoint = os.getenv("HEAPSTER_SERVICE_HOST") heapster_port = os.getenv("HEAPSTER_SERVICE_PORT") url = "http://%s:%s/api/v1/model" % (heapster_endpoint, heapster_port) client = AsyncHTTPClient(force_instance=True) try: result = yield client.fetch(url, method="GET", raise_error=False) if not result.error: self.heapster_base_url = url finally: client.close() else: try: yield self.http_client.get("/api/v1/proxy/namespaces/kube-system/services/heapster/api/v1/model") self.heapster_base_url = "%s%s" % ( self.http_client._base_url, "/api/v1/proxy/namespaces/kube-system/services/heapster/api/v1/model") except HTTPError as http_error: logging.exception(http_error) logging.debug("Cannot access Heaptser API through proxy API")
async def get(self): http_client = AsyncHTTPClient() lang = self.get_argument('lang', 'en') try: self.set_header("Content-Type", "text/xml") url = ( "https://wmts.geo.admin.ch/EPSG/2056/" "1.0.0/WMTSCapabilities.xml?lang={}" ).format(lang) print(f"Fetching WMTS GetCapability: {url}") response = await http_client.fetch(url) print("Done.") self.write(response.body) except HTTPError as e: print("Error: " + str(e)) except Exception as e: # Other errors are possible, such as IOError. print("Error: " + str(e)) http_client.close()
def post_suggest(self, user_id: str, application_id: str, session_id: str, locale: str, context: dict, callback) -> str: self.logger.debug( "user_id=%s,application_id=%s,session_id=%s,locale=%s," "context=%s", user_id, application_id, session_id, locale, context) url = "%s?session_id=%s&application_id=%s&locale=%s" % ( SUGGEST_URL, session_id, application_id, locale) url += "&user_id=%s" % user_id if user_id is not None else "" try: request_body = {"context": context} http_client = AsyncHTTPClient() http_client.fetch(HTTPRequest(url=url, body=dumps(request_body), method="POST"), callback=callback) http_client.close() except HTTPError: self.logger.error("url=%s", url) raise
def xiamen_zyt_callback(order_id, result): if ';' in result: result = result.split(';')[0] body = json.dumps({ "channelNo": "yiqichong001", "orderno": order_id, "responsetime": "20160116073449", "resultcode": result }) http_client = AsyncHTTPClient() try: request_log.info('XIAMEN_ZYT CALLBACK\n%s', body) http_client.fetch('http://localhost:8899/callback/xiamen_zyt', method='POST', body=body) except Exception: request_log.exception('FAIL') finally: http_client.close()
class AsyncHTTPTestCase(AsyncTestCase): def setUp(self): self.stopped = False self.running = False self.failure = None self.stop_args = None super(AsyncHTTPTestCase, self).setUp() self.port = None self.http_client = AsyncHTTPClient(io_loop=self.io_loop) if hasattr(self, 'get_app'): self.app = self.get_app() self.http_server = HTTPServer(self.app, io_loop=self.io_loop, **self.get_httpserver_options()) self.http_server.listen(self.get_http_port()) def fetch(self, path, **kwargs): self.http_client.fetch(self.get_url(path), self.stop, **kwargs) return self.wait() def get_httpserver_options(self): return {} def get_http_port(self): if self.port is None: self.port = get_unused_port() return self.port def get_url(self, path): return 'http://localhost:%s%s' % (self.get_http_port(), path) def tearDown(self): self.http_server.stop() self.http_client.close() super(AsyncHTTPTestCase, self).tearDown()
def trafficweb_callback(order_id, result): if ';' in result: result = result.split(';')[0] data = { "order_id": "2016012710232187985", "client_order_id": order_id, "status": result, "desc": "\\u8ba2\\u8d2d\\u6210\\u529f" } body = json.dumps(data) http_client = AsyncHTTPClient() try: request_log.info('TRAFFICWEB CALLBACK\n%s', body) http_client.fetch('http://localhost:8899/callback/bitefeng', method='POST', body=body) except Exception: request_log.exception('FAIL') finally: http_client.close()
def ibumobile_callback(order_id, result): desc = '' if ';' in result: result, desc = result.split(';') body = { "orderNo": order_id, "resCode": result, "redMsg": desc, } body = urlencode(body) http_client = AsyncHTTPClient() try: request_log.info('IBUMOBILE CALLBACK\n%s', body) http_client.fetch('http://localhost:8899/callback/ibumobile', method='POST', body=body) except Exception: request_log.exception('FAIL') finally: http_client.close()
class AsyncHTTPTestCase(AsyncTestCase): '''A test case that starts up an HTTP server. Subclasses must override get_app(), which returns the tornado.web.Application (or other HTTPServer callback) to be tested. Tests will typically use the provided self.http_client to fetch URLs from this server. Example: class MyHTTPTest(AsyncHTTPTestCase): def get_app(self): return Application([('/', MyHandler)...]) def test_homepage(self): # The following two lines are equivalent to # response = self.fetch('/') # but are shown in full here to demonstrate explicit use # of self.stop and self.wait. self.http_client.fetch(self.get_url('/'), self.stop) response = self.wait() # test contents of response ''' def setUp(self): super(AsyncHTTPTestCase, self).setUp() self.__port = None self.http_client = AsyncHTTPClient(io_loop=self.io_loop) self._app = self.get_app() self.http_server = HTTPServer(self._app, io_loop=self.io_loop, **self.get_httpserver_options()) self.http_server.listen(self.get_http_port()) def get_app(self): """Should be overridden by subclasses to return a tornado.web.Application or other HTTPServer callback. """ raise NotImplementedError() def fetch(self, path, **kwargs): """Convenience method to synchronously fetch a url. The given path will be appended to the local server's host and port. Any additional kwargs will be passed directly to AsyncHTTPClient.fetch (and so could be used to pass method="POST", body="...", etc). """ self.http_client.fetch(self.get_url(path), self.stop, **kwargs) return self.wait() def get_httpserver_options(self): """May be overridden by subclasses to return additional keyword arguments for HTTPServer. """ return {} def get_http_port(self): """Returns the port used by the HTTPServer. A new port is chosen for each test. """ if self.__port is None: self.__port = get_unused_port() return self.__port def get_url(self, path): """Returns an absolute url for the given path on the test server.""" return 'http://localhost:%s%s' % (self.get_http_port(), path) def tearDown(self): self.http_server.stop() self.http_client.close() super(AsyncHTTPTestCase, self).tearDown()
def up_wo(handler, partner): product_info = handler.slave.hmget( 'private:wo:%s:%d' % (handler.area, handler.price), ['product_id', 'price']) product_id = product_info[0] price = product_info[1] activity_id = partner['activity_id'] handler.up_req_time = time.localtime() tsp = time.strftime("%Y%m%d%H%M%S", handler.up_req_time) order_id = tsp + '000' + handler.order_id[-3:] handler.up_order_id = order_id # key+orderNo+phoneNum+productId+ activityId+realPrice) token = wo_key(partner['key'] + order_id + handler.mobile + product_id + activity_id + price) query = ( 'orderNo={order_id}&productId={product_id}&activityId={activity_id}' '&phoneNum={mobile}&realPrice={price}&token={token}').format( order_id=order_id, product_id=product_id, activity_id=activity_id, mobile=handler.mobile, price=price, token=token) url = partner['url.order'] + '?' + query # print(handler.order_id + ":" + body) request_log.info('CALL_REQ %s', query, extra={'orderid': handler.order_id}) # call & wait response = None result = 99999 http_client = AsyncHTTPClient() try: response = yield http_client.fetch(url, method='GET') except HTTPError as http_error: request_log.error('CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id}) result = 60000 + http_error.code except Exception as e: request_log.error('CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id}) finally: http_client.close() handler.up_resp_time = time.localtime() # <-------------- if response and response.code == 200: body = response.body.decode('utf8') request_log.info('CALL_RESP %s', body, extra={'orderid': handler.order_id}) try: resp = json.loads(body) status = resp['status'] result = RESULT_MAPPING.get(status, status) handler.up_result = result except Exception as e: result = 99999 request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id}) if handler.up_result is None: handler.up_result = result if result in [0, 60599]: for _ in range(5): result = yield query_order(order_id, partner) if result in [1, 9]: break # wait yield tornado.gen.Task(IOLoop.instance().add_timeout, time.time() + 120) if result == 1: yield handler.callback('1') elif result == 9: pass return result
class CommandHelper: def __init__(self, config: ConfigHelper) -> None: self.server = config.get_server() self.debug_enabled = config.getboolean('enable_repo_debug', False) if self.debug_enabled: logging.warning("UPDATE MANAGER: REPO DEBUG ENABLED") shell_cmd: SCMDComp = self.server.lookup_component('shell_command') self.scmd_error = shell_cmd.error self.build_shell_command = shell_cmd.build_shell_command self.pkg_updater: Optional[PackageDeploy] = None AsyncHTTPClient.configure(None, defaults=dict(user_agent="Moonraker")) self.http_client = AsyncHTTPClient() self.github_request_cache: Dict[str, CachedGithubResponse] = {} # GitHub API Rate Limit Tracking self.gh_rate_limit: Optional[int] = None self.gh_limit_remaining: Optional[int] = None self.gh_limit_reset_time: Optional[float] = None # Update In Progress Tracking self.cur_update_app: Optional[str] = None self.cur_update_id: Optional[int] = None self.full_complete: bool = False def get_server(self) -> Server: return self.server def is_debug_enabled(self) -> bool: return self.debug_enabled def set_update_info(self, app: str, uid: int, full_complete: bool = True ) -> None: self.cur_update_app = app self.cur_update_id = uid self.full_complete = full_complete def set_full_complete(self, complete: bool = False): self.full_complete = complete def clear_update_info(self) -> None: self.cur_update_app = self.cur_update_id = None self.full_complete = False def is_app_updating(self, app_name: str) -> bool: return self.cur_update_app == app_name def is_update_busy(self) -> bool: return self.cur_update_app is not None def set_package_updater(self, updater: PackageDeploy) -> None: self.pkg_updater = updater def get_rate_limit_stats(self) -> Dict[str, Any]: return { 'github_rate_limit': self.gh_rate_limit, 'github_requests_remaining': self.gh_limit_remaining, 'github_limit_reset_time': self.gh_limit_reset_time, } async def init_api_rate_limit(self) -> None: url = "https://api.github.com/rate_limit" while 1: try: resp = await self.github_api_request(url, is_init=True) assert isinstance(resp, dict) core = resp['resources']['core'] self.gh_rate_limit = core['limit'] self.gh_limit_remaining = core['remaining'] self.gh_limit_reset_time = core['reset'] except Exception: logging.exception("Error Initializing GitHub API Rate Limit") await asyncio.sleep(30.) else: reset_time = time.ctime(self.gh_limit_reset_time) logging.info( "GitHub API Rate Limit Initialized\n" f"Rate Limit: {self.gh_rate_limit}\n" f"Rate Limit Remaining: {self.gh_limit_remaining}\n" f"Rate Limit Reset Time: {reset_time}, " f"Seconds Since Epoch: {self.gh_limit_reset_time}") break async def run_cmd(self, cmd: str, timeout: float = 20., notify: bool = False, retries: int = 1, env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, sig_idx: int = 1 ) -> None: cb = self.notify_update_response if notify else None scmd = self.build_shell_command(cmd, callback=cb, env=env, cwd=cwd) while retries: if await scmd.run(timeout=timeout, sig_idx=sig_idx): break retries -= 1 if not retries: raise self.server.error("Shell Command Error") async def run_cmd_with_response(self, cmd: str, timeout: float = 20., retries: int = 5, env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, sig_idx: int = 1 ) -> str: scmd = self.build_shell_command(cmd, None, env=env, cwd=cwd) result = await scmd.run_with_response(timeout, retries, sig_idx=sig_idx) return result async def github_api_request(self, url: str, is_init: Optional[bool] = False ) -> JsonType: if self.gh_limit_remaining == 0: curtime = time.time() assert self.gh_limit_reset_time is not None if curtime < self.gh_limit_reset_time: raise self.server.error( f"GitHub Rate Limit Reached\nRequest: {url}\n" f"Limit Reset Time: {time.ctime(self.gh_limit_remaining)}") if url in self.github_request_cache: cached_request = self.github_request_cache[url] etag: Optional[str] = cached_request.get_etag() else: cached_request = CachedGithubResponse() etag = None self.github_request_cache[url] = cached_request headers = {"Accept": "application/vnd.github.v3+json"} if etag is not None: headers['If-None-Match'] = etag retries = 5 while retries: try: fut = self.http_client.fetch( url, headers=headers, connect_timeout=5., request_timeout=5., raise_error=False) resp: HTTPResponse resp = await asyncio.wait_for(fut, 10.) except Exception: retries -= 1 if retries > 0: logging.exception( f"Error Processing GitHub API request: {url}") await asyncio.sleep(1.) continue etag = resp.headers.get('etag', None) if etag is not None: if etag[:2] == "W/": etag = etag[2:] logging.info( "GitHub API Request Processed\n" f"URL: {url}\n" f"Response Code: {resp.code}\n" f"Response Reason: {resp.reason}\n" f"ETag: {etag}") if resp.code == 403: raise self.server.error( f"Forbidden GitHub Request: {resp.reason}") elif resp.code == 304: logging.info(f"Github Request not Modified: {url}") return cached_request.get_cached_result() if resp.code != 200: retries -= 1 if not retries: raise self.server.error( f"Github Request failed: {resp.code} {resp.reason}") logging.info( f"Github request error, {retries} retries remaining") await asyncio.sleep(1.) continue # Update rate limit on return success if 'X-Ratelimit-Limit' in resp.headers and not is_init: self.gh_rate_limit = int(resp.headers['X-Ratelimit-Limit']) self.gh_limit_remaining = int( resp.headers['X-Ratelimit-Remaining']) self.gh_limit_reset_time = float( resp.headers['X-Ratelimit-Reset']) decoded = json.loads(resp.body) if etag is not None: cached_request.update_result(etag, decoded) return decoded raise self.server.error( f"Retries exceeded for GitHub API request: {url}") async def http_download_request(self, url: str, content_type: str, timeout: float = 180. ) -> bytes: retries = 5 while retries: try: fut = self.http_client.fetch( url, headers={"Accept": content_type}, connect_timeout=5., request_timeout=timeout) resp: HTTPResponse resp = await asyncio.wait_for(fut, timeout + 10.) except Exception: retries -= 1 logging.exception("Error Processing Download") if not retries: raise await asyncio.sleep(1.) continue return resp.body raise self.server.error( f"Retries exceeded for GitHub API request: {url}") async def streaming_download_request(self, url: str, dest: Union[str, pathlib.Path], content_type: str, size: int, timeout: float = 180. ) -> None: if isinstance(dest, str): dest = pathlib.Path(dest) retries = 5 while retries: dl = StreamingDownload(self, dest, size) try: fut = self.http_client.fetch( url, headers={"Accept": content_type}, connect_timeout=5., request_timeout=timeout, streaming_callback=dl.on_chunk_recd) resp: HTTPResponse resp = await asyncio.wait_for(fut, timeout + 10.) except Exception: retries -= 1 logging.exception("Error Processing Download") if not retries: raise await asyncio.sleep(1.) continue finally: await dl.close() if resp.code < 400: return def notify_update_response(self, resp: Union[str, bytes], is_complete: bool = False ) -> None: if self.cur_update_app is None: return resp = resp.strip() if isinstance(resp, bytes): resp = resp.decode() done = is_complete and self.full_complete notification = { 'message': resp, 'application': self.cur_update_app, 'proc_id': self.cur_update_id, 'complete': done} self.server.send_event( "update_manager:update_response", notification) async def install_packages(self, package_list: List[str], **kwargs ) -> None: if self.pkg_updater is None: return await self.pkg_updater.install_packages(package_list, **kwargs) def close(self) -> None: self.http_client.close()
class UpdateManager: def __init__(self, config): self.server = config.get_server() self.config = config self.config.read_supplemental_config(SUPPLEMENTAL_CFG_PATH) self.repo_debug = config.getboolean('enable_repo_debug', False) auto_refresh_enabled = config.getboolean('enable_auto_refresh', False) self.distro = config.get('distro', "debian").lower() if self.distro not in SUPPORTED_DISTROS: raise config.error(f"Unsupported distro: {self.distro}") if self.repo_debug: logging.warn("UPDATE MANAGER: REPO DEBUG ENABLED") env = sys.executable mooncfg = self.config[f"update_manager static {self.distro} moonraker"] self.updaters = { "system": PackageUpdater(self), "moonraker": GitUpdater(self, mooncfg, MOONRAKER_PATH, env) } self.current_update = None # TODO: Check for client config in [update_manager]. This is # deprecated and will be removed. client_repo = config.get("client_repo", None) if client_repo is not None: client_path = config.get("client_path") name = client_repo.split("/")[-1] self.updaters[name] = WebUpdater(self, { 'repo': client_repo, 'path': client_path }) client_sections = self.config.get_prefix_sections( "update_manager client") for section in client_sections: cfg = self.config[section] name = section.split()[-1] if name in self.updaters: raise config.error("Client repo named %s already added" % (name, )) client_type = cfg.get("type") if client_type == "git_repo": self.updaters[name] = GitUpdater(self, cfg) elif client_type == "web": self.updaters[name] = WebUpdater(self, cfg) else: raise config.error("Invalid type '%s' for section [%s]" % (client_type, section)) # GitHub API Rate Limit Tracking self.gh_rate_limit = None self.gh_limit_remaining = None self.gh_limit_reset_time = None self.gh_init_evt = Event() self.cmd_request_lock = Lock() self.is_refreshing = False # Auto Status Refresh self.last_auto_update_time = 0 self.refresh_cb = None if auto_refresh_enabled: self.refresh_cb = PeriodicCallback(self._handle_auto_refresh, UPDATE_REFRESH_INTERVAL_MS) self.refresh_cb.start() AsyncHTTPClient.configure(None, defaults=dict(user_agent="Moonraker")) self.http_client = AsyncHTTPClient() self.server.register_endpoint("/machine/update/moonraker", ["POST"], self._handle_update_request) self.server.register_endpoint("/machine/update/klipper", ["POST"], self._handle_update_request) self.server.register_endpoint("/machine/update/system", ["POST"], self._handle_update_request) self.server.register_endpoint("/machine/update/client", ["POST"], self._handle_update_request) self.server.register_endpoint("/machine/update/status", ["GET"], self._handle_status_request) # Register Ready Event self.server.register_event_handler("server:klippy_identified", self._set_klipper_repo) # Initialize GitHub API Rate Limits and configured updaters IOLoop.current().spawn_callback(self._initalize_updaters, list(self.updaters.values())) async def _initalize_updaters(self, initial_updaters): self.is_refreshing = True await self._init_api_rate_limit() for updater in initial_updaters: if isinstance(updater, PackageUpdater): ret = updater.refresh(False) else: ret = updater.refresh() if asyncio.iscoroutine(ret): await ret self.is_refreshing = False async def _set_klipper_repo(self): kinfo = self.server.get_klippy_info() if not kinfo: logging.info("No valid klippy info received") return kpath = kinfo['klipper_path'] env = kinfo['python_path'] kupdater = self.updaters.get('klipper', None) if kupdater is not None and kupdater.repo_path == kpath and \ kupdater.env == env: # Current Klipper Updater is valid return kcfg = self.config[f"update_manager static {self.distro} klipper"] self.updaters['klipper'] = GitUpdater(self, kcfg, kpath, env) await self.updaters['klipper'].refresh() async def _check_klippy_printing(self): klippy_apis = self.server.lookup_plugin('klippy_apis') result = await klippy_apis.query_objects({'print_stats': None}, default={}) pstate = result.get('print_stats', {}).get('state', "") return pstate.lower() == "printing" async def _handle_auto_refresh(self): if await self._check_klippy_printing(): # Don't Refresh during a print logging.info("Klippy is printing, auto refresh aborted") return cur_time = time.time() cur_hour = time.localtime(cur_time).tm_hour time_diff = cur_time - self.last_auto_update_time # Update packages if it has been more than 12 hours # and the local time is between 12AM and 5AM if time_diff < MIN_REFRESH_TIME or cur_hour >= MAX_PKG_UPDATE_HOUR: # Not within the update time window return self.last_auto_update_time = cur_time vinfo = {} need_refresh_all = not self.is_refreshing async with self.cmd_request_lock: self.is_refreshing = True try: for name, updater in list(self.updaters.items()): if need_refresh_all: ret = updater.refresh() if asyncio.iscoroutine(ret): await ret if hasattr(updater, "get_update_status"): vinfo[name] = updater.get_update_status() except Exception: logging.exception("Unable to Refresh Status") return finally: self.is_refreshing = False uinfo = { 'version_info': vinfo, 'github_rate_limit': self.gh_rate_limit, 'github_requests_remaining': self.gh_limit_remaining, 'github_limit_reset_time': self.gh_limit_reset_time, 'busy': self.current_update is not None } self.server.send_event("update_manager:update_refreshed", uinfo) async def _handle_update_request(self, web_request): if await self._check_klippy_printing(): raise self.server.error("Update Refused: Klippy is printing") app = web_request.get_endpoint().split("/")[-1] if app == "client": app = web_request.get('name') inc_deps = web_request.get_boolean('include_deps', False) if self.current_update is not None and \ self.current_update[0] == app: return f"Object {app} is currently being updated" updater = self.updaters.get(app, None) if updater is None: raise self.server.error(f"Updater {app} not available") async with self.cmd_request_lock: self.current_update = (app, id(web_request)) try: await updater.update(inc_deps) except Exception as e: self.notify_update_response(f"Error updating {app}") self.notify_update_response(str(e), is_complete=True) raise finally: self.current_update = None return "ok" async def _handle_status_request(self, web_request): check_refresh = web_request.get_boolean('refresh', False) # Don't refresh if a print is currently in progress or # if an update is in progress. Just return the current # state if self.current_update is not None or \ await self._check_klippy_printing(): check_refresh = False need_refresh = False if check_refresh: # If there is an outstanding request processing a # refresh, we don't need to do it again. need_refresh = not self.is_refreshing await self.cmd_request_lock.acquire() self.is_refreshing = True vinfo = {} try: for name, updater in list(self.updaters.items()): await updater.check_initialized(120.) if need_refresh: ret = updater.refresh() if asyncio.iscoroutine(ret): await ret if hasattr(updater, "get_update_status"): vinfo[name] = updater.get_update_status() except Exception: raise finally: if check_refresh: self.is_refreshing = False self.cmd_request_lock.release() return { 'version_info': vinfo, 'github_rate_limit': self.gh_rate_limit, 'github_requests_remaining': self.gh_limit_remaining, 'github_limit_reset_time': self.gh_limit_reset_time, 'busy': self.current_update is not None } async def execute_cmd(self, cmd, timeout=10., notify=False, retries=1): shell_command = self.server.lookup_plugin('shell_command') cb = self.notify_update_response if notify else None scmd = shell_command.build_shell_command(cmd, callback=cb) while retries: if await scmd.run(timeout=timeout, verbose=notify): break retries -= 1 if not retries: raise self.server.error("Shell Command Error") async def execute_cmd_with_response(self, cmd, timeout=10.): shell_command = self.server.lookup_plugin('shell_command') scmd = shell_command.build_shell_command(cmd, None) result = await scmd.run_with_response(timeout, retries=5) if result is None: raise self.server.error(f"Error Running Command: {cmd}") return result async def _init_api_rate_limit(self): url = "https://api.github.com/rate_limit" while 1: try: resp = await self.github_api_request(url, is_init=True) core = resp['resources']['core'] self.gh_rate_limit = core['limit'] self.gh_limit_remaining = core['remaining'] self.gh_limit_reset_time = core['reset'] except Exception: logging.exception("Error Initializing GitHub API Rate Limit") await tornado.gen.sleep(30.) else: reset_time = time.ctime(self.gh_limit_reset_time) logging.info( "GitHub API Rate Limit Initialized\n" f"Rate Limit: {self.gh_rate_limit}\n" f"Rate Limit Remaining: {self.gh_limit_remaining}\n" f"Rate Limit Reset Time: {reset_time}, " f"Seconds Since Epoch: {self.gh_limit_reset_time}") break self.gh_init_evt.set() async def github_api_request(self, url, etag=None, is_init=False): if not is_init: timeout = time.time() + 30. try: await self.gh_init_evt.wait(timeout) except Exception: raise self.server.error("Timeout while waiting for GitHub " "API Rate Limit initialization") if self.gh_limit_remaining == 0: curtime = time.time() if curtime < self.gh_limit_reset_time: raise self.server.error( f"GitHub Rate Limit Reached\nRequest: {url}\n" f"Limit Reset Time: {time.ctime(self.gh_limit_remaining)}") headers = {"Accept": "application/vnd.github.v3+json"} if etag is not None: headers['If-None-Match'] = etag retries = 5 while retries: try: timeout = time.time() + 10. fut = self.http_client.fetch(url, headers=headers, connect_timeout=5., request_timeout=5., raise_error=False) resp = await tornado.gen.with_timeout(timeout, fut) except Exception: retries -= 1 msg = f"Error Processing GitHub API request: {url}" if not retries: raise self.server.error(msg) logging.exception(msg) await tornado.gen.sleep(1.) continue etag = resp.headers.get('etag', None) if etag is not None: if etag[:2] == "W/": etag = etag[2:] logging.info("GitHub API Request Processed\n" f"URL: {url}\n" f"Response Code: {resp.code}\n" f"Response Reason: {resp.reason}\n" f"ETag: {etag}") if resp.code == 403: raise self.server.error( f"Forbidden GitHub Request: {resp.reason}") elif resp.code == 304: logging.info(f"Github Request not Modified: {url}") return None if resp.code != 200: retries -= 1 if not retries: raise self.server.error( f"Github Request failed: {resp.code} {resp.reason}") logging.info( f"Github request error, {retries} retries remaining") await tornado.gen.sleep(1.) continue # Update rate limit on return success if 'X-Ratelimit-Limit' in resp.headers and not is_init: self.gh_rate_limit = int(resp.headers['X-Ratelimit-Limit']) self.gh_limit_remaining = int( resp.headers['X-Ratelimit-Remaining']) self.gh_limit_reset_time = float( resp.headers['X-Ratelimit-Reset']) decoded = json.loads(resp.body) decoded['etag'] = etag return decoded async def http_download_request(self, url): retries = 5 while retries: try: timeout = time.time() + 130. fut = self.http_client.fetch( url, headers={"Accept": "application/zip"}, connect_timeout=5., request_timeout=120.) resp = await tornado.gen.with_timeout(timeout, fut) except Exception: retries -= 1 logging.exception("Error Processing Download") if not retries: raise await tornado.gen.sleep(1.) continue return resp.body def notify_update_response(self, resp, is_complete=False): resp = resp.strip() if isinstance(resp, bytes): resp = resp.decode() notification = { 'message': resp, 'application': None, 'proc_id': None, 'complete': is_complete } if self.current_update is not None: notification['application'] = self.current_update[0] notification['proc_id'] = self.current_update[1] self.server.send_event("update_manager:update_response", notification) def close(self): self.http_client.close() if self.refresh_cb is not None: self.refresh_cb.stop()
class TestMockHTTP(AsyncTestCase): def setUp(self): super(TestMockHTTP, self).setUp() self.mock = MockHTTP() self.http_client = AsyncHTTPClient(io_loop=self.io_loop) def tearDown(self): self.mock.shutdown() self.http_client.close() super(TestMockHTTP, self).tearDown() def fetch(self, path, **kwargs): """Convenience method to synchronously fetch a url. The given path will be appended to the local server's host and port. Any additional kwargs will be passed directly to `.AsyncHTTPClient.fetch` (and so could be used to pass ``method="POST"``, ``body="..."``, etc). """ self.http_client.fetch( HTTPRequest(url=self.mock.get_url(path), **kwargs), callback=self.stop) return self.wait() def test_get_request(self): """Tests a get request that expects nothing to return but an 200.""" self.mock.expects(method=GET, path='/index.html') resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.code, 200) self.assert_(self.mock.verify()) def test_get_request_wrong_url(self): """Tests a get request that expects a different URL.""" self.mock.expects(method=GET, path='/index.html') resp = self.fetch(method="GET", path="notindex.html") self.assertEqual(resp.code, 404) self.assertRaises(UnexpectedURLException, self.mock.verify) def test_get_with_code(self): self.mock.expects(method=GET, path='/index.html').will(http_code=500) resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.code, 500, 'Expected 500 response.') self.assert_(self.mock.verify()) def test_get_with_body(self): """Tests a get request that returns a different URL.""" test_body = 'Test response.' self.mock.expects(method=GET, path='/index.html').will(body=test_body) resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.body, test_body) self.assert_(self.mock.verify()) def test_get_with_header(self): """Tests a get request that includes a custom header.""" test_header_name = 'Content-Type' test_header_contents = 'text/html' self.mock.expects(method=GET, path='/index.html').will( headers={test_header_name: test_header_contents}) resp = self.fetch(method="GET", path="index.html") self.assertTrue(resp.headers[test_header_name].startswith(test_header_contents)) self.assert_(self.mock.verify()) def test_multiple_get(self): """Test getting a URL twice.""" self.mock.expects(method=GET, path='/index.html') resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.code, 200) resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.code, 200) self.assert_(self.mock.verify()) def test_never_get(self): """Test a URL that has a 'never' times on it.""" self.mock.expects(method=GET, path='/index.html', times=never) resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.code, 404) self.assertRaises(UnexpectedURLException, self.mock.verify) def test_get_once_got_twice(self): """Test getting a URL twice that expects to be retrieved once only.""" self.mock.expects(method=GET, path='/index.html', times=once) resp = self.fetch(method="GET", path="index.html") resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.code, 404) self.assertRaises(AlreadyRetrievedURLException, self.mock.verify) def test_get_once_got_never(self): """Test never getting a URL that expects to be retrieved once only.""" self.mock.expects(method=GET, path='/index.html', times=once) self.assertRaises(UnretrievedURLException, self.mock.verify) def test_get_5times_got_never(self): """Test never getting a URL that expects to be retrieved 5 times only.""" self.mock.expects(method=GET, path='/index.html', times=5) self.assertRaises(UnretrievedURLException, self.mock.verify) def test_get_5times_got_5times(self): """Test getting 5 times a URL that expects to be retrieved 5 times""" self.mock.expects(method=GET, path='/index.html', times=5) call = lambda: self.fetch('/index.html') [ call() for x in xrange(5) ] self.mock.verify() def test_get_at_least_once_got_twice(self): """Test getting a URL twice that expects to be retrieved at least once.""" self.mock.expects(method=GET, path='/index.html', times=at_least_once) resp = self.fetch(method="GET", path="index.html") self.assertEqual(resp.code, 200) self.assert_(self.mock.verify()) def test_get_at_least_once_got_never(self): """Test never getting a URL that expects to be retrieved at least once.""" self.mock.expects(method=GET, path='/index.html', times=at_least_once) self.assertRaises(UnretrievedURLException, self.mock.verify) def test_get_after(self): """Test two URLs that expect to be retrieved in order.""" test_body = 'Test POST body.\r\n' headers = {'content-type': 'text/plain'} self.mock.expects(method=GET, path='/index.html', name='url #1') self.mock.expects(method=POST, path='/index.html', after='url #1', headers=headers, body=test_body) self.fetch(method="GET", path="index.html") self.fetch(method="POST", path="index.html", body = test_body, headers=headers) self.assert_(self.mock.verify()) def test_get_after_wrong_order(self): """Test two URLs that expect to be retrieved in order, but aren't.""" test_body = 'Test POST body.\r\n' headers = {'content-type': 'text/plain'} self.mock.expects(method=GET, path='/index.html', name='url #1') self.mock.expects(method=POST, path='/index.html', after='url #1', body=test_body, headers=headers) resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = headers) self.assertEqual(resp.code, 404) self.assertRaises(URLOrderingException, self.mock.verify) def test_post(self): """Tests a POST request.""" test_body = 'Test POST body.\r\n' headers = {'content-type': 'text/plain'} self.mock.expects(method=POST, path='/index.html', body=test_body, headers=headers).will(http_code=201) resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = headers) self.assertEqual(resp.code, 201) self.assert_(self.mock.verify()) def test_post_bad_body(self): """Tests a POST request that sends the wrong body data.""" test_body = 'Test POST body.\r\n' expected_body = 'Expected POST body.\r\n' headers = {'content-type': 'text/plain'} self.mock.expects(method=POST, path='/index.html', body=expected_body) resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = headers) self.assertEqual(resp.code, 404) self.assertRaises(WrongBodyException, self.mock.verify) def test_post_header(self): """Tests a POST request with some custom headers.""" test_body = 'Test POST body.\r\n' test_headers = {'content-type': 'application/atom+xml; type=entry', 'content-length': str(len(test_body)), 'Slug': 'ooze',} self.mock.expects(method=POST, path='/index.html', body=test_body, headers=test_headers).will(http_code=201) resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers) self.assertEqual(resp.code, 201) self.assert_(self.mock.verify()) def test_post_unexpected_header(self): """Tests a POST request with an unexpected header.""" test_body = 'Test POST body.\r\n' test_headers = {'content-type': 'application/atom+xml; type=entry', 'content-length': str(len(test_body)), 'Slug': 'ooze',} expected_headers = {'content-type': 'application/atom+xml; type=entry', 'content-length': str(len(test_body)),} self.mock.expects(method=POST, path='/index.html', body=test_body, headers=expected_headers) resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers) self.assertEqual(resp.code, 200) self.assert_(self.mock.verify()) def test_post_missing_header(self): """Tests a POST request without an expected header.""" test_body = 'Test POST body.\r\n' test_headers = {'content-type': 'application/atom+xml; type=entry', 'content-length': str(len(test_body)),} expected_headers = {'content-type': 'application/atom+xml; type=entry', 'content-length': str(len(test_body)), 'Slug': 'ooze',} self.mock.expects(method=POST, path='/index.html', body=test_body, headers=expected_headers) resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers) self.assertEqual(resp.code, 404) self.assertRaises(WrongHeaderException, self.mock.verify) def test_post_unexpected_header_value(self): """Tests a POST request with an unexpected header value.""" test_body = 'Test POST body.\r\n' test_headers = {'content-type': 'application/atom+xml; type=entry', 'content-length': str(len(test_body)), 'Slug': 'ooze',} expected_headers = {'content-type': 'application/atom+xml; type=entry', 'content-length': str(len(test_body)), 'Slug': 'slime',} self.mock.expects(method=POST, path='/index.html', body=test_body, headers=expected_headers) resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers) self.assertEqual(resp.code, 404) self.assertRaises(WrongHeaderValueException, self.mock.verify)
class TestClient(object): def __init__(self, app, raise_handler_exc=False): # TODO add `host` kwarg if app.is_running: raise RuntimeError('You should not instance TestClient' 'when applicaion is running') self.raise_handler_exc = raise_handler_exc # start::tornado.testing.AsyncTestCase self.__stopped = False self.__running = False self.__failure = None self.__stop_args = None self.__timeout = None # end::tornado.testing.AsyncTestCase self._handler_exc_info = None app.update_settings({ 'TESTING': True, 'PROCESSES': 1 }) # redo setup to make settings changes effective app.setup() self.app = app self.setup() def setup(self): """Setup facilities for running the server NOTE if you get all "Address already in use" error except the first one, the bug may well happened in this function. """ # start::tornado.testing.AsyncTestCase #self.io_loop = self.get_new_ioloop() # end::tornado.testing.AsyncTestCase #self.app.io_loop = self.io_loop # init app.io_loop, app.application and app.http_server self.app._init_application() self.app.make_http_server() self.io_loop = self.app.io_loop if self.raise_handler_exc: self.patch_app_handlers() self.http_server = self.app.http_server self.http_client = AsyncHTTPClient(io_loop=self.io_loop) def close(self): """CLose http_server, io_loop by sequence, to ensure the environment is cleaned up and invoking `setup` successfully within next test function It is suggested to be called in `TestCase.tearDown` """ # start::tornado.testing.AsyncHTTPTestCase self.http_server.stop() if (not IOLoop.initialized() or self.http_client.io_loop is not IOLoop.instance()): self.http_client.close() # end::tornado.testing.AsyncHTTPTestCase # start::tornado.testing.AsyncTestCase if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()): # Try to clean up any file descriptors left open in the ioloop. # This avoids leaks, especially when tests are run repeatedly # in the same process with autoreload (because curl does not # set FD_CLOEXEC on its file descriptors) self.io_loop.close(all_fds=True) # end::tornado.testing.AsyncTestCase def _parse_cookies(self, resp): if COOKIE_HEADER_KEY in resp.headers: c = SimpleCookie(resp.headers.get(COOKIE_HEADER_KEY)) elif COOKIE_HEADER_KEY.lower() in resp.headers: c = SimpleCookie(resp.headers.get(COOKIE_HEADER_KEY.lower())) else: c = None resp.cookies = c def _add_cookies(self, cookies, kwgs): # SimpleCookie is inherited from dict, so judge it first if isinstance(cookies, SimpleCookie): c = cookies elif isinstance(cookies, dict): c = SimpleCookie() for k, v in cookies.items(): c[k] = v else: raise TypeError('cookies kwarg should be dict or SimpleCookie instance') if 'headers' in kwgs: headers = kwgs['headers'] else: headers = kwgs['headers'] = {} headers['Cookie'] = c.output().lstrip(COOKIE_HEADER_KEY + ': ') #print 'headers', headers def stop(self, _arg=None, **kwargs): '''Stops the ioloop, causing one pending (or future) call to wait() to return. Keyword arguments or a single positional argument passed to stop() are saved and will be returned by wait(). ''' assert _arg is None or not kwargs self.__stop_args = kwargs or _arg if self.__running: self.io_loop.stop() self.__running = False self.__stopped = True def wait(self, condition=None, timeout=5): """Runs the IOLoop until stop is called or timeout has passed. In the event of a timeout, an exception will be thrown. If condition is not None, the IOLoop will be restarted after stop() until condition() returns true. """ if not self.__stopped: if timeout: def timeout_func(): try: raise self.failureException( 'Async operation timed out after %s seconds' % timeout) except Exception: self.__failure = sys.exc_info() self.stop() if self.__timeout is not None: self.io_loop.remove_timeout(self.__timeout) self.__timeout = self.io_loop.add_timeout(time.time() + timeout, timeout_func) while True: self.__running = True with NullContext(): # Wipe out the StackContext that was established in # self.run() so that all callbacks executed inside the # IOLoop will re-run it. self.io_loop.start() if (self.__failure is not None or condition is None or condition()): break assert self.__stopped self.__stopped = False self.__rethrow() result = self.__stop_args self.__stop_args = None return result def __rethrow(self): if self.__failure is not None: failure = self.__failure self.__failure = None raise_exc_info(failure) def request(self, method, path, data=None, json=False, files=None, cookies=None, **kwgs): kwgs['method'] = method ## `path` should be utf-8 encoded string to complete requote process #if isinstance(path, str): # path = path.encode('utf8') path = requote_uri(path) # `body` must be passed if method is one of those three if method in ['POST', 'PUT', 'PATCH']: headers = kwgs.setdefault('headers', {}) body = '' if files: boundary = '1234567890' headers['Content-Type'] = 'multipart/form-data; boundary=%s' % boundary L = [] if data: for k, v in data.items(): L.append('--' + boundary) L.append('Content-Disposition: form-data; name="%s"' % k) L.append('') L.append(v) for k, f in files.items(): L.append('--' + boundary) L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (k, f[0])) L.append('Content-Type: %s' % mimetypes.guess_type(f[0])[0] or 'application/octet-stream') L.append('') L.append(f[1]) L.append('--%s--' % boundary) L.append('') body = '\r\n'.join(L) else: if data: if json: body = json_encode(data) headers['Content-Type'] = 'application/json' else: headers['Content-Type'] = 'application/x-www-form-urlencoded' body = urlencode(data) kwgs['body'] = body else: if data: path = '%s?%s' % (path, urlencode(data)) if cookies: self._add_cookies(cookies, kwgs) #print 'fetch kwgs', kwgs url = self.get_url(path) logging.debug('testing fetch url: %s', url) self.http_client.fetch(url, self.stop, **kwgs) resp = self.wait() self._parse_cookies(resp) return resp def get(self, *args, **kwgs): return self.request('GET', *args, **kwgs) def delete(self, *args, **kwgs): return self.request('DELETE', *args, **kwgs) def post(self, *args, **kwgs): return self.request('POST', *args, **kwgs) def put(self, *args, **kwgs): return self.request('PUT', *args, **kwgs) def patch(self, *args, **kwgs): return self.request('PATCH', *args, **kwgs) def get_protocol(self): return 'http' def get_http_port(self): return self.app.settings['PORT'] def get_url(self, path): """Returns an absolute url for the given path on the test server.""" return '%s://localhost:%s%s' % (self.get_protocol(), self.get_http_port(), path) def get_handler_exc(self): if self._handler_exc_info: raise_exc_info(self._handler_exc_info) def patch_handler(self, hdr): if not isinstance(hdr, web.StaticFileHandler): hdr._testing_app_client = self hdr._handle_request_exception = _handle_request_exception def patch_app_handlers(self): for host_pattern, rules in self.app.application.handlers: for r in rules: self.patch_handler(r.handler_class)
def post(self, path): if 'admin-point' not in self.current_user['roles']: self.finish() return user_id = self.json_args['user_id'] _type = self.json_args['type'] operator = self.json_args['operator'] token = self.json_args['token'] income = self.json_args['income'] notes = self.json_args['notes'] if _type == 'refund-manual': order_id = self.json_args['order_id'] try: request_log.info('CHECK REFUND ORDER %s', order_id) order_coll = self.application.glados_client.GLaDOS.order order = yield order_coll.find_one({'_id': order_id}) if order is None: raise RuntimeError('无法查询订单') if order['user_id'] != user_id: raise RuntimeError('您查询的订单与选择的用户不一致,请核实。') if order['value'] != int(float(income) * 10000): # print(int(float(income) * 10000)) # print(order.value) raise RuntimeError('订单金额异常') if order['back_result'] not in ['1', '00000']: raise RuntimeError('订单状态不是正常完成状态') t = datetime.strptime(order_id[1:9], '%Y%m%d') request_log.info('CHECK REFUND FROM %s', str(t)) trans_coll = self.application.glados_client.GLaDOS.transaction cursor = trans_coll.find({ 'order_id': order_id, 'user_id': user_id, 'type': 'refund-manual', 'create_time': { '$gt': t } }) refund_trans = yield cursor.to_list(10) if refund_trans: return self.finish( json.dumps({ 'status': 'fail', 'msg': '该订单已经存在手工退款,请核实!' })) request_log.info('CHECK REFUND ORDER %s DONE', order_id) except RuntimeError as re: return self.finish( json.dumps({ 'status': 'fail', 'msg': str(re) })) except Exception as e: request_log.exception('FUND FAIL') return self.finish(json.dumps({'status': 'fail'})) else: order_id = '' sign0 = signature( '{user_id}{order_id}{income}{operator}{token}'.format( user_id=user_id, order_id=order_id, income=income, operator=operator, token=token)) body = json.dumps({ 'type': _type, 'order_id': order_id, 'user_id': user_id, 'income': income, 'operator': operator, 'token': token, 'notes': notes, 'sign': sign0 }) http_client = AsyncHTTPClient() try: downstream = self.application.config['downstream'][user_id] url = downstream['shard'] msg = {'status': 'fail'} request = HTTPRequest(url="http://%s/admin/fund" % url, method='POST', body=body, headers={'Content-Type': 'application/json'}) response = yield http_client.fetch(request) if response.code == 200: request_log.info(response.body) msg = json.loads(response.body.decode('utf8')) if msg['status'] == 'ok': msg = { 'status': 'ok', 'msg': '财务操作成功,用户当前余额为' + str(int(msg['msg']) / 10000) } except Exception as e: request_log.exception('FUND FAIL2') msg = {'status': 'fail', 'msg': str(e)} finally: http_client.close() self.finish(json.dumps(msg))
def up_cmcc_sx(handler, partner): handler.up_req_time = time.localtime() appKey = partner["appKey"] timeStamp = (datetime.now() + timedelta(seconds=0)).strftime("%Y%m%d%H%M%S") login_no = partner["login_no"] acc_nbr = partner["acc_nbr"] group_id = partner["group_id"] member_role_id = partner["member_role_id"] gprs_accept = group_id + timeStamp + "000001" prod_prcid = None k = 'private:cmcc_sx:{carrier}:{price}'.format(carrier=handler.carrier, price=handler.price) prod_prcid = handler.slave.get(k) if prod_prcid is None: handler.up_result = 5003 return handler.up_result data = ('acc_nbr={acc_nbr}&' 'appKey={appKey}&' 'gprs_accept={gprs_accept}&' 'group_id={group_id}&' 'login_no={login_no}&' 'member_role_id={member_role_id}&' 'prod_prcid={prod_prcid}&' 'timeStamp={timeStamp}').format(appKey=appKey, timeStamp=timeStamp, login_no=login_no, acc_nbr=acc_nbr, prod_prcid=prod_prcid, group_id=group_id, member_role_id=member_role_id, gprs_accept=gprs_accept) sign = to_md5(quote(data)) rsa_sign = to_para(sign) body = '&'.join([ "acc_nbr=%s" % quote(acc_nbr), "appKey=%s" % quote(appKey), "gprs_accept=%s" % quote(gprs_accept), "group_id=%s" % quote(group_id), "login_no=%s" % quote(login_no), "member_role_id=%s" % quote(member_role_id), "prod_prcid=%s" % quote(prod_prcid), "timeStamp=%s" % quote(timeStamp), "sign=%s" % quote(rsa_sign), ]) url = partner['url_auth'] url = url + '?' + body h = {'Content-Type': 'application/json; charset=utf-8'} result = 9999 up_result = None http_client = AsyncHTTPClient() try: request_log.info("REQU %s", body, extra={'orderid': handler.order_id}) response = yield http_client.fetch(url, method='GET', headers=h, request_timeout=120) except HTTPError as http_error: request_log.error('CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id}) result = 60000 + http_error.code response = None except Exception as e: request_log.error('CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id}) response = None finally: http_client.close() handler.up_resp_time = time.localtime() if response and response.code == 200: response_body = response.body.decode('utf8') request_log.info("RESP %s", response_body, extra={'orderid': handler.order_id}) try: response_body = json.loads(response_body) up_result = response_body.get("resCode") sp_order_id = response_body.get("orderId") handler.up_order_id = sp_order_id result = RESULT_MAP.get(up_result, 9) if result == 1: handler.back_result = result yield handler.callback('1') elif result == 9: pass handler.up_result = up_result if handler.up_result == '0000000': handler.master.set( "map:cmcc_sx:{sp_order_id}".format( sp_order_id=sp_order_id), handler.order_id) except Exception as e: result = 9999 handler.up_result = result request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id}) return result
def up_21cn(handler, partner): """ telecom -> up_order """ key = '21cn:%d' % handler.price if 'prefix' in partner: key = 'private:%s:%s' % (partner['prefix'], key) if handler.scope: key = '%s:%s' % (key, handler.scope) prod = handler.slave.hmget(key, ['plat_offer_id', 'activity_id']) handler.up_req_time = time.localtime() tsp = time.strftime("%Y%m%d%H%M%S", handler.up_req_time) passphrase = partner['pass'] iv = partner['iv'] # {"request_no":"201410211021070000000113", # "service_code":"FS0001", # "contract_id":"100052", # "order_id":"0", # "plat_offer_id":"100151", 100151:100033 # "phone_id":"18905172668", # "channel_id":"1", # "activity_id":"100033"} code = json.dumps({ 'request_no': handler.order_id, 'service_code': 'FS0001', 'contract_id': partner['contract_id'], 'order_id': '0', 'plat_offer_id': prod[0], 'phone_id': handler.mobile, 'channel_id': '1', 'activity_id': prod[1], }) request_log.info('CALL_CODE %s', code, extra={'orderid': handler.order_id}) # aes aes = AES.new(passphrase, AES.MODE_CBC, iv) bytes = aes.encrypt(pad(code)) encrypted = hexlify(bytes) body = json.dumps({'partner_no': partner['partner_no'], 'code': encrypted}) url = partner['url.order'] # print(handler.order_id + ":" + body) request_log.info('CALL_REQ %s', body, extra={'orderid': handler.order_id}) response = None result = 99999 # call & wait http_client = AsyncHTTPClient() try: response = yield http_client.fetch(url, method='POST', headers=h, body=body) except HTTPError as http_error: request_log.error('CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id}) result = 60000 + http_error.code except Exception as e: request_log.error('CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id}) finally: http_client.close() handler.up_resp_time = time.localtime() # <-------------- if response and response.code == 200: body = response.body.decode('utf8') request_log.info('CALL_RESP %s', body, extra={'orderid': handler.order_id}) try: # {"request_no":"201410211021070000000113","result_code":"00000"} resp = json.loads(body) result = int(resp['result_code']) handler.up_order_id = resp['request_no'] handler.up_cost = handler.cost handler.up_result = result except Exception as e: result = 99999 request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id}) if handler.up_result is None: handler.up_result = result return result
def up_seek(handler, partner): slave = handler.slave # private data if handler.scope is None: k = 'private:seek:{area}:{price}'.format(area=handler.area, price=handler.price) else: k = 'private:seek:{area}:{price}:{scope}'.format(area=handler.area, price=handler.price, scope=handler.scope) prod = slave.hmget(k, ['prod_id', 'sum']) if prod[0] is None: return 99999 prod_id = prod[0] prod_sum = prod[1] handler.up_req_time = time.localtime() tsp = int(time.mktime(handler.up_req_time)) back_url = urllib.parse.quote_plus(partner['callback']) sign = ''.join([ 'appkey', partner['key'], 'channelOrderNo', handler.order_id, 'customer', handler.mobile, 'notifyUrl', back_url, 'prodId', prod_id, 'prodNum1prodPayType0', 'sum', prod_sum, 'timestamp', str(tsp), partner['secret'] ]) sign = signature(sign).lower() q = '&'.join([ 'appkey=' + partner['key'], 'prodId=' + prod_id, 'customer=' + handler.mobile, 'sum=' + prod_sum, 'prodNum=1', 'prodPayType=0', 'channelOrderNo=' + handler.order_id, 'notifyUrl=' + back_url, 'timestamp=' + str(tsp), 'sign=' + sign ]) url = partner['url.order'] # print(handler.order_id + ":" + body) full_url = url + '/r/Channel/createOrder?' + q request_log.info('CALL_REQ %s', full_url, extra={'orderid': handler.order_id}) response = None result = 99999 # call & wait http_client = AsyncHTTPClient() try: response = yield http_client.fetch(full_url, method='GET') except HTTPError as http_error: request_log.error('CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id}) result = 60000 + http_error.code except Exception as e: request_log.error('CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id}) finally: http_client.close() if response and response.code == 200: body = response.body.decode('utf8') request_log.info('CALL_RESP1 %s', body, extra={'orderid': handler.order_id}) try: # {"orderNo":"1418189340100004","orderStatus":0,"createOrderTime":"20141210132900","resultCode":"1000","resultReason":""} resp = json.loads(body) result = int(resp['resultCode']) if result == 1000: handler.up_order_id = resp['orderNo'] except Exception as e: result = 99999 request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id}) if result == 1000 and handler.up_order_id: # SUBMIT tsp = int(time.mktime(handler.up_req_time)) sign = ''.join([ 'appkey' + partner['key'], 'orderNo' + handler.up_order_id, 'timestamp' + str(tsp), partner['secret'] ]) sign = signature(sign).lower() q = '&'.join([ 'orderNo=' + handler.up_order_id, 'appkey=' + partner['key'], 'timestamp=' + str(tsp), 'sign=' + sign ]) full_url = url + '/r/Channel/submitOrder?' + q response = None result = 99999 # call & wait http_client = AsyncHTTPClient() try: response = yield http_client.fetch(full_url, method='GET') except HTTPError as http_error: request_log.error('CALL UPSTREAM2 FAIL %s', http_error, extra={'orderid': handler.order_id}) result = 60000 + http_error.code except Exception as e: request_log.error('CALL UPSTREAM2 FAIL %s', e, extra={'orderid': handler.order_id}) finally: http_client.close() if response and response.code == 200: body = response.body.decode('utf8') request_log.info('CALL_RESP2 %s', body, extra={'orderid': handler.order_id}) try: # {"orderNo":"1418189340100004","orderStatus":0,"createOrderTime":"20141210132900","resultCode":"1000","resultReason":""} resp = json.loads(body) result = int(resp['resultCode']) except Exception as e: result = 99999 request_log.error('PARSE UPSTREAM2 %s', e, extra={'orderid': handler.order_id}) # result mapping result = RESULT_MAPPING.get(result, result) handler.up_result = result handler.up_resp_time = time.localtime() # <-------------- return result
class AsyncHTTPTestCase(AsyncTestCase): def initialize_ioloop(self): self.io_loop = self.get_new_ioloop() self.http_client = AsyncHTTPClient(io_loop=self.io_loop) def setup(self): self.stopped = False self.running = False self.failure = None self.stop_args = None if 'get_app' in dir(self.__class__): self.initialize_ioloop() self.app = self.get_app() elif hasattr(self, 'get_handler_spec') and self.get_handler_spec: spec = self.get_handler_spec() if spec: if isinstance(spec, tuple): if len(spec) == 3: (pattern, handler, kwargs) = spec elif len(spec) == 2: (pattern, handler) = spec kwargs = {} elif isinstance(spec, URLSpec): pattern = spec.regex.pattern handler = spec.handler_class kwargs = spec.kwargs # create an isolated version of the handler self.isolated_handler = type('IsolatedHandler', (handler,), {}) self.initialize_ioloop() self.app = Application([ (pattern, self.isolated_handler, kwargs) ], self.get_application_settings()) if self.app: self.port = get_unused_port() self.http_server = HTTPServer( self.app, io_loop=self.io_loop, **self.get_httpserver_options() ) self.http_server.listen(self.port, address="0.0.0.0") def fetch(self, path, **kwargs): """ Simple wrapper around ``http_client``. If the given ``path`` doesn't start with 'http' than ``path`` is passed on to the ``http_client.fetch`` """ self.http_client.fetch(self.get_url(path), self.stop, **kwargs) return self.wait() def get_httpserver_options(self): return {} def get_url(self, path): if not path.startswith('http'): return 'http://localhost:%s%s' % (self.port, path) return path def teardown(self): if 'http_server' in dir(self.__class__): self.http_server.stop() if 'http_client' in dir(self.__class__): self.http_client.close()
class AsyncHTTPTestCase(AsyncTestCase): '''A test case that starts up an HTTP server. Subclasses must override get_app(), which returns the tornado.web.Application (or other HTTPServer callback) to be tested. Tests will typically use the provided self.http_client to fetch URLs from this server. Example:: class MyHTTPTest(AsyncHTTPTestCase): def get_app(self): return Application([('/', MyHandler)...]) def test_homepage(self): # The following two lines are equivalent to # response = self.fetch('/') # but are shown in full here to demonstrate explicit use # of self.stop and self.wait. self.http_client.fetch(self.get_url('/'), self.stop) response = self.wait() # test contents of response ''' def setUp(self): super(AsyncHTTPTestCase, self).setUp() self.__port = None self.http_client = AsyncHTTPClient(io_loop=self.io_loop) self._app = self.get_app() self.http_server = HTTPServer(self._app, io_loop=self.io_loop, **self.get_httpserver_options()) self.http_server.listen(self.get_http_port(), address="127.0.0.1") def get_app(self): """Should be overridden by subclasses to return a tornado.web.Application or other HTTPServer callback. """ raise NotImplementedError() def fetch(self, path, **kwargs): """Convenience method to synchronously fetch a url. The given path will be appended to the local server's host and port. Any additional kwargs will be passed directly to AsyncHTTPClient.fetch (and so could be used to pass method="POST", body="...", etc). """ self.http_client.fetch(self.get_url(path), self.stop, **kwargs) return self.wait() def get_httpserver_options(self): """May be overridden by subclasses to return additional keyword arguments for HTTPServer. """ return {} def get_http_port(self): """Returns the port used by the HTTPServer. A new port is chosen for each test. """ if self.__port is None: self.__port = get_unused_port() return self.__port def get_url(self, path): """Returns an absolute url for the given path on the test server.""" return 'http://localhost:%s%s' % (self.get_http_port(), path) def tearDown(self): self.http_server.stop() self.http_client.close() super(AsyncHTTPTestCase, self).tearDown()
class CommandHelper: def __init__(self, config): self.server = config.get_server() self.debug_enabled = config.getboolean('enable_repo_debug', False) if self.debug_enabled: logging.warn("UPDATE MANAGER: REPO DEBUG ENABLED") shell_command = self.server.lookup_plugin('shell_command') self.build_shell_command = shell_command.build_shell_command AsyncHTTPClient.configure(None, defaults=dict(user_agent="Moonraker")) self.http_client = AsyncHTTPClient() # GitHub API Rate Limit Tracking self.gh_rate_limit = None self.gh_limit_remaining = None self.gh_limit_reset_time = None self.gh_init_evt = Event() # Update In Progress Tracking self.cur_update_app = self.cur_update_id = None def get_server(self): return self.server def is_debug_enabled(self): return self.debug_enabled def set_update_info(self, app, uid): self.cur_update_app = app self.cur_update_id = uid def clear_update_info(self): self.cur_update_app = self.cur_update_id = None def is_app_updating(self, app_name): return self.cur_update_app == app_name def is_update_busy(self): return self.cur_update_app is not None def get_rate_limit_stats(self): return { 'github_rate_limit': self.gh_rate_limit, 'github_requests_remaining': self.gh_limit_remaining, 'github_limit_reset_time': self.gh_limit_reset_time, } async def init_api_rate_limit(self): url = "https://api.github.com/rate_limit" while 1: try: resp = await self.github_api_request(url, is_init=True) core = resp['resources']['core'] self.gh_rate_limit = core['limit'] self.gh_limit_remaining = core['remaining'] self.gh_limit_reset_time = core['reset'] except Exception: logging.exception("Error Initializing GitHub API Rate Limit") await tornado.gen.sleep(30.) else: reset_time = time.ctime(self.gh_limit_reset_time) logging.info( "GitHub API Rate Limit Initialized\n" f"Rate Limit: {self.gh_rate_limit}\n" f"Rate Limit Remaining: {self.gh_limit_remaining}\n" f"Rate Limit Reset Time: {reset_time}, " f"Seconds Since Epoch: {self.gh_limit_reset_time}") break self.gh_init_evt.set() async def run_cmd(self, cmd, timeout=10., notify=False, retries=1, env=None): cb = self.notify_update_response if notify else None scmd = self.build_shell_command(cmd, callback=cb, env=env) while retries: if await scmd.run(timeout=timeout): break retries -= 1 if not retries: raise self.server.error("Shell Command Error") async def run_cmd_with_response(self, cmd, timeout=10., env=None): scmd = self.build_shell_command(cmd, None, env=env) result = await scmd.run_with_response(timeout, retries=5) if result is None: raise self.server.error(f"Error Running Command: {cmd}") return result async def github_api_request(self, url, etag=None, is_init=False): if not is_init: timeout = time.time() + 30. try: await self.gh_init_evt.wait(timeout) except Exception: raise self.server.error("Timeout while waiting for GitHub " "API Rate Limit initialization") if self.gh_limit_remaining == 0: curtime = time.time() if curtime < self.gh_limit_reset_time: raise self.server.error( f"GitHub Rate Limit Reached\nRequest: {url}\n" f"Limit Reset Time: {time.ctime(self.gh_limit_remaining)}") headers = {"Accept": "application/vnd.github.v3+json"} if etag is not None: headers['If-None-Match'] = etag retries = 5 while retries: try: timeout = time.time() + 10. fut = self.http_client.fetch(url, headers=headers, connect_timeout=5., request_timeout=5., raise_error=False) resp = await tornado.gen.with_timeout(timeout, fut) except Exception: retries -= 1 msg = f"Error Processing GitHub API request: {url}" if not retries: raise self.server.error(msg) logging.exception(msg) await tornado.gen.sleep(1.) continue etag = resp.headers.get('etag', None) if etag is not None: if etag[:2] == "W/": etag = etag[2:] logging.info("GitHub API Request Processed\n" f"URL: {url}\n" f"Response Code: {resp.code}\n" f"Response Reason: {resp.reason}\n" f"ETag: {etag}") if resp.code == 403: raise self.server.error( f"Forbidden GitHub Request: {resp.reason}") elif resp.code == 304: logging.info(f"Github Request not Modified: {url}") return None if resp.code != 200: retries -= 1 if not retries: raise self.server.error( f"Github Request failed: {resp.code} {resp.reason}") logging.info( f"Github request error, {retries} retries remaining") await tornado.gen.sleep(1.) continue # Update rate limit on return success if 'X-Ratelimit-Limit' in resp.headers and not is_init: self.gh_rate_limit = int(resp.headers['X-Ratelimit-Limit']) self.gh_limit_remaining = int( resp.headers['X-Ratelimit-Remaining']) self.gh_limit_reset_time = float( resp.headers['X-Ratelimit-Reset']) decoded = json.loads(resp.body) decoded['etag'] = etag return decoded async def http_download_request(self, url): retries = 5 while retries: try: timeout = time.time() + 130. fut = self.http_client.fetch( url, headers={"Accept": "application/zip"}, connect_timeout=5., request_timeout=120.) resp = await tornado.gen.with_timeout(timeout, fut) except Exception: retries -= 1 logging.exception("Error Processing Download") if not retries: raise await tornado.gen.sleep(1.) continue return resp.body def notify_update_response(self, resp, is_complete=False): resp = resp.strip() if isinstance(resp, bytes): resp = resp.decode() notification = { 'message': resp, 'application': self.cur_update_app, 'proc_id': self.cur_update_id, 'complete': is_complete } self.server.send_event("update_manager:update_response", notification) def close(self): self.http_client.close()
def up_suopai(handler, partner): handler.up_req_time = time.localtime() userNo = partner['userNo'] userKey = partner['userKey'] mobile = handler.mobile orderId = handler.order_id k = 'private:suopai:{carrier}:{price}'.format(carrier=handler.carrier, price=handler.price) productNo = handler.slave.get(k) if productNo is None: handler.up_result = 5003 return handler.up_result body = { "userNo": userNo, "userKey": userKey, "mobile": mobile, "productNo": productNo, "orderId": orderId, } url = partner['url.order'] requ_body = urlencode(body) url = url + '?' + requ_body result = 9999 http_client = AsyncHTTPClient() try: request_log.info("SUOPAI REQ %s", requ_body, extra={'orderid': handler.order_id}) response = yield http_client.fetch(url, method='GET', request_timeout=120) except HTTPError as http_error: request_log.error('SUOPAI CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id}) result = 60000 + http_error.code response = None except Exception as e: request_log.error('SUOPAI CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id}) response = None finally: http_client.close() # 订单超时处理 if result == 60599: jobId = yield query_jobid(handler, partner) if jobId is None: result = 9 else: response = yield query_order_by_jobid(handler, partner, jobId) handler.up_resp_time = time.localtime() if response and response.code == 200: response_body = response.body.decode('utf8') request_log.info("RESP %s", response_body, extra={'orderid': handler.order_id}) try: response_body = json.loads(response_body) up_result = response_body.get('success') sp_order_id = response_body.get("jobId") handler.up_order_id = sp_order_id result = RESULT_MAP.get(up_result, 9) handler.up_result = up_result if handler.up_result == 'true': handler.master.set( "map:suopai:{sp_order_id}".format(sp_order_id=sp_order_id), handler.order_id) except Exception as e: result = 9999 handler.up_result = result request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id}) return result
def post(self, *args, **kwargs): self.set_header('Content-Type', 'application/json') user_id = self.get_argument("user_id", None) session_id = self.get_argument("session_id", None) context_id = self.get_argument("context_id", None) application_id = self.get_argument("application_id", None) product_id = self.get_argument("product_id", None) _type = self.get_argument("type", None) body = json_decode(self.request.body) if application_id is None: self.set_status(412) self.finish( json_encode( { "status": "error", "message": "missing param=application_id" } ) ) elif session_id is None: self.set_status(412) self.finish( json_encode( { "status": "error", "message": "missing param=session_id" } ) ) elif product_id is None: self.set_status(412) self.finish( json_encode( { "status": "error", "message": "missing param=product_id" } ) ) else: url = "%s/feedback?session_id=%s&application_id=%s&product_id=%s&type=%s" % ( CONTEXT_URL, session_id, application_id, product_id, _type ) if user_id is not None: url += "&user_id=%s" % user_id if context_id is not None: url += "&context_id=%s" % context_id # http_client = HTTPClient() # # response = http_client.fetch( # HTTPRequest( # url=url, # body=json_encode(body), # method="POST" # ) # ) # # http_client.close() # self.set_status(response.status) # self.finish(response.body) http_client = AsyncHTTPClient() http_client.fetch( HTTPRequest( url=url, body=json_encode(body), method="POST" ), self.context_feedback_response_handler ) http_client.close()
class AsyncHTTPTestCase(AsyncTestCase): """A test case that starts up an HTTP server. Subclasses must override get_app(), which returns the tornado.web.Application (or other HTTPServer callback) to be tested. Tests will typically use the provided self.http_client to fetch URLs from this server. Example:: class MyHTTPTest(AsyncHTTPTestCase): def get_app(self): return Application([('/', MyHandler)...]) def test_homepage(self): # The following two lines are equivalent to # response = self.fetch('/') # but are shown in full here to demonstrate explicit use # of self.stop and self.wait. self.http_client.fetch(self.get_url('/'), self.stop) response = self.wait() # test contents of response """ __test__ = False @testify.setup def build_client(self): self.http_client = AsyncHTTPClient(io_loop=self.io_loop) @testify.setup def build_server(self): self.__port = None # We have a much simplified implementation of cookies for our test hook. # We're going to ignore pretty much everything like host, path, secure, expires etc. # It's basically a dumb key-value store, represented by.... a dictionary self.cookie_jar = {} self._app = self.get_app() self.http_server = HTTPServer(self._app, io_loop=self.io_loop, **self.get_httpserver_options()) self.http_server.listen(self.get_http_port(), address="127.0.0.1") @testify.teardown def stop_client_server(self): self.http_client.close() self.http_server.stop() def get_app(self): """Should be overridden by subclasses to return a tornado.web.Application or other HTTPServer callback. """ raise NotImplementedError() def fetch(self, request, **kwargs): """Convenience method to synchronously fetch a url. The given path will be appended to the local server's host and port. Any additional kwargs will be passed directly to AsyncHTTPClient.fetch (and so could be used to pass method="POST", body="...", etc). """ timeout = None if 'timeout' in kwargs: timeout = kwargs.pop('timeout') if hasattr(request, 'url'): parsed = urlparse.urlparse(request.url) request.url = self.update_urlparsed(parsed) else: parsed = urlparse.urlparse(request) request = self.update_urlparsed(parsed) if 'headers' in kwargs and not isinstance(kwargs['headers'], tornado.httputil.HTTPHeaders): # Upgrade our headers to a fancy object so we can add multiple values. kwargs['headers'] = tornado.httputil.HTTPHeaders(kwargs['headers']) else: kwargs.setdefault('headers', tornado.httputil.HTTPHeaders()) if 'body' in kwargs and not isinstance(kwargs['body'], basestring): kwargs['body'] = json.dumps(kwargs['body']) kwargs['headers']['Content-Type'] = 'application/json' if 'cookies' in kwargs: for cookie in kwargs['cookies']: for val in cookie.values(): kwargs['headers'].add('Cookie', val.OutputString(None)) if self.cookie_jar: cookie = Cookie.SimpleCookie(self.cookie_jar) for val in cookie.values(): kwargs['headers'].add('Cookie', val.OutputString(None)) self.http_client.fetch(request, self.stop, **kwargs) res = self.wait(timeout=timeout) for cookie_val in res.headers.get_list('set-cookie'): cookie = Cookie.SimpleCookie() cookie.load(cookie_val) for val in cookie.values(): if not val.value: del self.cookie_jar[val.key] else: self.cookie_jar[val.key] = val.value return TestHTTPResponse(res) def set_cookie(self, name, value): self.cookie_jar[name] = value def clear_cookie(self, name): del self.cookie_jar[name] def get_httpserver_options(self): """May be overridden by subclasses to return additional keyword arguments for HTTPServer. """ return {} def get_http_port(self): """Returns the port used by the HTTPServer. A new port is chosen for each test. """ if self.__port is None: self.__port = get_unused_port() return self.__port def get_url(self, path): """Returns an absolute url for the given path on the test server.""" return 'http://localhost:%s%s' % (self.get_http_port(), path) def update_urlparsed(self, parsed): return urlparse.urlunparse(['http', 'localhost:%s' % self.get_http_port()] + list(parsed[2:]))