def post_suggest(self, user_id: str, application_id: str, session_id: str, locale: str, context: dict,
callback) -> str:
self.logger.debug(
"user_id=%s,application_id=%s,session_id=%s,locale=%s,"
"context=%s",
user_id, application_id, session_id, locale, context
)
url = "%s?session_id=%s&application_id=%s&locale=%s" % (
SUGGEST_URL, session_id, application_id, locale
)
url += "&user_id=%s" % user_id if user_id is not None else ""
try:
request_body = {
"context": context
}
http_client = AsyncHTTPClient()
http_client.fetch(HTTPRequest(url=url, body=dumps(request_body), method="POST"), callback=callback)
http_client.close()
except HTTPError:
self.logger.error("url=%s", url)
raise
def post_context_message(
self, context_id: str, direction: int, message_text: str, callback, detection: dict = None, now=None):
"""
Direction is 1 user 0 jemboo
:type direction: int
"""
self.logger.debug(
"context_id=%s,direction=%s,message_text=%s,detection=%s",
context_id, direction, message_text, detection
)
now = datetime.now() if now is None else now
try:
request_body = {
"direction": direction,
"text": message_text,
"created": now.isoformat()
}
if detection is not None:
request_body["detection"] = detection
url = "%s/%s/messages/" % (CONTEXT_URL, context_id)
http_client = AsyncHTTPClient()
http_client.fetch(
HTTPRequest(url=url, method="POST", body=json_encode(request_body)),
callback=callback
)
http_client.close()
except HTTPError as e:
self.logger.error("post_context_message,url=%s", url)
raise
def _check_heapster_status(settings):
if "HEAPSTER_SERVICE_HOST" in settings:
heapster_endpoint = settings["HEAPSTER_SERVICE_HOST"]
heapster_port = settings["HEAPSTER_SERVICE_PORT"]
endpoint = "http://%s:%s/api/v1/model" % (heapster_endpoint, heapster_port)
headers = dict()
else:
kubernetes_endpoint = settings["KUBERNETES_SERVICE_HOST"]
if not kubernetes_endpoint.startswith("http"):
kubernetes_endpoint = "https://%s:%s" % (kubernetes_endpoint, settings["KUBERNETES_SERVICE_PORT"])
endpoint = "%s%s" % (
kubernetes_endpoint,
"/api/v1/proxy/namespaces/kube-system/services/heapster/api/v1/model"
)
headers = dict(Authorization="Bearer %s" % settings['token'])
client = AsyncHTTPClient(force_instance=True)
try:
result = yield client.fetch(
endpoint + "/metrics",
method="GET",
validate_cert=False,
headers=headers,
raise_error=False)
if not result.error:
raise Return(status_ok())
else:
raise Return(status_error(result.error))
finally:
client.close()
def post(self, *args, **kwargs):
# TODO
# save the messages to local database
# file = self.request.files['image'][0]
# file_name = file["filename"]
# image = file['body']
text = self.get_argument("text")
data = {
"from": "Mailgun Sandbox <*****@*****.**>",
"to": "<*****@*****.**>",
"subject": "Hello Udaan",
"text": text,
}
data = urlencode(data)
client = AsyncHTTPClient()
headers_object = HTTPHeaders({"X-Mailgun-Variables": dumps({"X-Mailgun-Variables": {"password": "******"}})})
request_object = HTTPRequest("https://api.mailgun.net/v3/sandbox1713f24a60034b5ab5e7fa0ca2faa9b6.mailgun.org"
"/messages",
method="POST",
headers=headers_object,
body=data,
auth_username="******",
auth_password="******"
)
print(request_object.headers.get_list("X-Mailgun-Variables"))
response = yield client.fetch(request_object)
client.close()
print(response)
if response.code == 200:
msg = "email send successfully"
self.respond(msg, response.code)
else:
msg = "Please try again"
self.respond(msg, response.code)
def request(self, url, method="GET", **kwargs):
client = AsyncHTTPClient(force_instance=True)
try:
result = yield client.fetch(url, method=method, headers=self.build_headers(), **kwargs)
raise Return(result)
finally:
client.close()
class Worker(object):
# Seconds before retrying a request to server in seconds
POLL_INTERVAL = 1
# Seconds to wait when there are no new jobs
NO_JOB_WAIT= 60
def __init__(self, api_url, mongo_uri, **kwargs):
self.url = api_url
self.name = socket.gethostname()
self.http_client = AsyncHTTPClient()
self.models = {"keras": KerasModel(**kwargs)}
self.data_path = os.path.join(os.path.dirname(__file__), "data")
self.data_loader = DataLoader(api_url, mongo_uri)
def json_decode(self, body):
json_body = json.loads(body.decode("utf-8"))
# This needs to be called two times on escaped json string. ¯\_(ツ)_/¯
if not isinstance(json_body, dict):
return json.loads(json_body)
return json_body
@gen.coroutine
def get_new_job(self):
try:
response = yield self.http_client.fetch(self.url + "/worker/" + self.name + "/job")
job = self.json_decode(response.body)
# no jobs, wait 1 min before next request.
if "wait" in job:
print "Experiment %s is done! There are no new jobs yet." % job["experiment_id"]
IOLoop.current().call_later(self.NO_JOB_WAIT, lambda: self.get_new_job())
return
# if job type is unsupported (only keras supported at the moment)
if job["model_type"] not in self.models:
raise ValueError("Only the following models are supported right now: " + ", ".join(self.models.keys()))
# TODO: clean it up! terrible, I know..
model = self.models[job["model_type"]]
model.exp_id = job["experiment_id"]
job["model_params"]["data_filename"] = job["data_filename"]
result = model.run_job(job["model_params"])
# Get a new job from server
IOLoop.current().call_later(1, lambda: self.get_new_job())
except HTTPError as e:
print("Error: " + str(e))
IOLoop.current().call_later(self.POLL_INTERVAL, lambda: self.get_new_job())
except Exception as e:
# Other errors are possible, such as IOError.
print("Error: " + str(e))
IOLoop.current().call_later(self.POLL_INTERVAL, lambda: self.get_new_job())
def close(self):
self.http_client.close()
class AsyncHTTPTestCase(AsyncTestCase):
'''A test case that starts up an HTTP server.
Subclasses must override get_app(), which returns the
tornado.web.Application (or other HTTPServer callback) to be tested.
Tests will typically use the provided self.http_client to fetch
URLs from this server.
Example:
class MyHTTPTest(AsyncHTTPTestCase):
def get_app(self):
return Application([('/', MyHandler)...])
def test_homepage(self):
self.http_client.fetch(self.get_url('/'), self.stop)
response = self.wait()
# test contents of response
'''
def setUp(self):
super(AsyncHTTPTestCase, self).setUp()
self.__port = None
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
self._app = self.get_app()
self.http_server = HTTPServer(self._app, io_loop=self.io_loop,
**self.get_httpserver_options())
self.http_server.listen(self.get_http_port())
def get_app(self):
"""Should be overridden by subclasses to return a
tornado.web.Application or other HTTPServer callback.
"""
raise NotImplementedError()
def get_httpserver_options(self):
"""May be overridden by subclasses to return additional
keyword arguments for HTTPServer.
"""
return {}
def get_http_port(self):
"""Returns the port used by the HTTPServer.
A new port is chosen for each test.
"""
if self.__port is None:
self.__port = get_unused_port()
return self.__port
def get_url(self, path):
"""Returns an absolute url for the given path on the test server."""
return 'http://localhost:%s%s' % (self.get_http_port(), path)
def tearDown(self):
self.http_server.stop()
self.http_client.close()
super(AsyncHTTPTestCase, self).tearDown()
def get(self, url_path, **kwargs):
params = self.build_params(url_path, **kwargs)
url = url_concat(self.build_url(url_path, **kwargs), params)
client = AsyncHTTPClient(force_instance=True)
try:
result = yield client.fetch(url, method="GET", headers=self.build_headers())
raise Return(result)
finally:
client.close()
def get_detect(self, location: str, callback) -> dict:
self.logger.debug("location=%s", location)
url = "%s%s" % (DETECT_URL, location)
try:
http_client = AsyncHTTPClient()
http_client.fetch(HTTPRequest(url=url, method="GET"), callback=callback)
http_client.close()
except HTTPError as e:
self.logger.error("get_detect,url=%s", url)
raise
def delete(self, url_path, **kwargs):
client = AsyncHTTPClient(force_instance=True)
try:
response = yield client.fetch(
self.build_url(url_path, **kwargs),
method="DELETE",
headers=self.build_headers())
raise Return(response)
finally:
client.close()
def get_context_messages(self, handler: WebSocketHandler, callback) -> dict:
try:
if handler.context is None or handler.context["_rev"] != handler.context_rev:
self.logger.debug(
"get_context_from_service,context_id=%s,_rev=%s", str(handler.context_id), handler.context_rev)
http_client = AsyncHTTPClient()
url = "%s/%s/messages" % (CONTEXT_URL, str(handler.context_id))
http_client.fetch(HTTPRequest(url=url, method="GET"), callback=callback)
http_client.close()
except HTTPError as e:
self.logger.error("get_context,url=%s", url)
raise
def async_fetch(self, url, method=HTTP_METHOD_GET, params=None, callback=None):
"""
"""
if not callback:
raise HttpClientCallbackError, "callback is wrong"
# 先打包数据
request = self._pack_data(url, method, params)
# 发起请求
http_client = AsyncHTTPClient()
http_client.fetch(request, callback=callback)
http_client.close()
def coroutine_fetch(self, url, method=HTTP_METHOD_GET, params=None):
"""
"""
# 打包数据
request = self._pack_data(url, method, params)
http_client = AsyncHTTPClient()
response = yield http_client.fetch(request)
http_client.close()
if not response:
yield response
return
yield response.body
def request(self, url_path, method="GET", **kwargs):
params = self.build_params(url_path, **kwargs)
if not urlparse.urlparse(url_path).netloc:
url = url_concat(self.build_url(url_path, **kwargs), params)
else:
url = url_concat(url_path, params)
client = AsyncHTTPClient(force_instance=True)
try:
result = yield client.fetch(url, method=method, headers=self.build_headers())
raise Return(result)
finally:
client.close()
def run_client():
client = AsyncHTTPClient()
range_it = xrange(loops)
t0 = perf.perf_counter()
for _ in range_it:
futures = [client.fetch(url) for j in xrange(CONCURRENCY)]
for fut in futures:
resp = yield fut
buf = resp.buffer
buf.seek(0, 2)
assert buf.tell() == len(CHUNK) * NCHUNKS
namespace['dt'] = perf.perf_counter() - t0
client.close()
def patch(self, url_path, **kwargs):
url = self.build_url(url_path, **kwargs)
params = self.build_params(url_path, **kwargs)
client = AsyncHTTPClient(force_instance=True)
try:
result = yield client.fetch(
url,
method="PATCH",
headers=self.build_headers("application/merge-patch+json"),
**params)
raise Return(result)
finally:
client.close()
def __call__(self, message):
http_client = AsyncHTTPClient(force_instance=self._force_instance)
debug('client = %s, max_buffer_size = %s', http_client.__class__.__name__,
getattr(http_client, 'max_buffer_size') if hasattr(http_client, 'max_buffer_size') else 'unsupported')
response = ('', None)
try:
response = yield http_client.fetch(self.url, body=message, method='POST',
connect_timeout=self.connection_timeout,
request_timeout=self.connection_timeout)
except HTTPError as e:
error("HTTP Error: %s, payload size (bytes): %s", e, len(message))
handle_exception(ErrorMessage.from_exception(e, address=self.url))
finally:
if self._force_instance:
http_client.close()
raise gen.Return(response)
def send_async_request(url, headers=None, method="GET", body=None):
headers = headers or {}
if body or method.upper() == "POST":
if 'Content-Type' not in headers:
headers["Content-Type"] = "application/x-www-form-urlencoded"
req = httpclient.HTTPRequest(
url, method=method, body=body, headers=headers, allow_nonstandard_methods=True)
http_request = AsyncHTTPClient()
response = ""
try:
response = yield http_request.fetch(req)
except httpclient.HTTPError as e:
print("Error:" + str(e))
except Exception as e:
print("Error:" + str(e))
else:
return response
finally:
http_request.close()
class AsyncHTTPTestCase(AsyncTestCase):
def setup(self):
self.stopped = False
self.running = False
self.failure = None
self.stop_args = None
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
if 'get_app' in dir(self.__class__):
# the app needs its own io loop
self.io_loop = self.get_new_ioloop()
self.port = get_unused_port()
self.app = self.get_app()
self.http_server = HTTPServer(self.app, io_loop=self.io_loop,
**self.get_httpserver_options())
self.http_server.listen(self.port)
else:
# if we don't test handlers use the clients io loop
self.io_loop = self.http_client.io_loop
def fetch(self, path, **kwargs):
"""
Simple wrapper around ``http_client``. If the given ``path`` doesn't
start with 'http' than ``path`` is passed on to the
``http_client.fetch``
"""
self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
return self.wait()
def get_httpserver_options(self):
return {}
def get_url(self, path):
if not path.startswith('http'):
return 'http://localhost:%s%s' % (self.port, path)
return path
def teardown(self):
if 'http_server' in dir(self.__class__):
self.http_server.stop()
if 'http_client' in dir(self.__class__):
self.http_client.close()
def run_client():
client = AsyncHTTPClient()
range_it = xrange(loops)
if loops == -1:
# -1 is forever
range_it = iter(int, 1)
else:
range_it = xrange(loops)
for _ in range_it:
futures = [client.fetch(url) for j in xrange(CONCURRENCY)]
for fut in futures:
resp = yield fut
buf = resp.buffer
buf.seek(0, 2)
assert buf.tell() == len(CHUNK) * NCHUNKS
time.sleep(sleep)
client.close()
def xicheng_callback(order_id, result):
desc = ''
if ';' in result:
result, desc = result.split(';')
# cust_id + echo + secret + tsp
sign = signature64('njqxwlkj' + '2016011422063442' + 'njqxwl@QX' + '20160114220634')
body = BODY.format(sign=sign, order_id=order_id, result=result, desc=desc)
http_client = AsyncHTTPClient()
try:
request_log.info('XICHENG CALLBACK\n%s', body)
h = {'Content-Type': 'application/xml;charset=UTF-8'}
http_client.fetch('http://localhost:8899/callback/xicheng', headers=h, method='POST', body=body)
except Exception:
request_log.exception('FAIL')
finally:
http_client.close()
def query_order_by_jobid(handler, partner, jobId):
userNo = partner['userNo']
userKey = partner['userKey']
query_body = {
"userNo": userNo,
"userKey": userKey,
"jobId": jobId,
}
url = partner['url.query2']
request_body = urlencode(query_body)
url = url + '?' + request_body
http_client = AsyncHTTPClient()
try:
request_log.info("SUOPAI REQ %s",
request_body,
extra={'orderid': handler.order_id})
response = yield http_client.fetch(url,
method='GET',
request_timeout=120)
except HTTPError as http_error:
request_log.error('SUOPAI CALL UPSTREAM FAIL %s',
http_error,
extra={'orderid': handler.order_id})
# result = 60000 + http_error.code
response = None
except Exception as e:
request_log.error('SUOPAI CALL UPSTREAM FAIL %s',
e,
extra={'orderid': handler.order_id})
response = None
finally:
http_client.close()
return response
def post(self, product):
args = self.json_args
if 'admin' not in self.current_user['roles'] or 'user_id' not in args:
args['user_id'] = self.current_user['partner_id']
if 'blocking' in self.application.config and 'url' in self.application.config[
'blocking']:
http_client = AsyncHTTPClient()
try:
url = '%s/query/%s/export' % (
self.application.config['blocking']['url'], product)
request = HTTPRequest(url=url,
method='POST',
body=json.dumps(args),
request_timeout=120)
response = yield http_client.fetch(request)
if response.code == 200:
body = response.body.decode('utf8')
self.finish(body)
else:
self.send_error(500)
except Exception as e:
request_log.exception('QUERY CHAIN')
self.send_error(500)
finally:
http_client.close()
else: # fallback
o = OrderBlockingQueryHandler(self.application, self.request)
o.json_args = args
o.post(product)
self._headers = o._headers
self._status_code = o._status_code
self._write_buffer = o._write_buffer
self.finish()
def get_suggestion_items(self, user_id: str, application_id: str, session_id: str, locale: str, suggestion_id: str,
page_size: int, offset: int, callback):
self.logger.debug(
"user_id=%s,application_id=%s,session_id=%s,locale=%s,"
"suggestion_id=%s,page_size=%s,offset=%s",
user_id, application_id, session_id, locale, suggestion_id, page_size, offset
)
url = "%s/%s/items?session_id=%s&application_id=%s&locale=%s&page_size=%s&offset=%s" % (
SUGGEST_URL, suggestion_id, session_id, application_id, locale, page_size, offset
)
url += "&user_id=%s" % user_id if user_id is not None else ""
try:
http_client = AsyncHTTPClient()
http_client.fetch(HTTPRequest(url=url, method="GET"), callback=callback)
http_client.close()
except HTTPError:
self.logger.error("url=%s", url)
raise
class from_web_stream(Stream):
def __init__(self,
url='http://127.0.0.1:9999',
read_timeout=60 * 60 * 24,
start=True,
**kwargs):
self.url = url
self.request_timeout = read_timeout
self.http_client = AsyncHTTPClient()
super(from_web_stream, self).__init__(ensure_io_loop=True, **kwargs)
self.stopped = True
if start:
self.start()
@gen.coroutine
def get(self):
# if self.read_timeout is None:
# HTTPRequest._DEFAULTS['request_timeout']=None
# defaultset none is 20s,hack源代码也无法解决的，这部分代码不管用
requests = HTTPRequest(url=self.url,
streaming_callback=self.on_chunk,
request_timeout=self.request_timeout)
yield self.http_client.fetch(requests)
@gen.coroutine
def on_chunk(self, chunk):
chunk >> self
def start(self):
if self.stopped:
self.stopped = False
if self.http_client is None:
self.http_client = AsyncHTTPClient()
self.get()
def stop(self):
if self.http_client is not None:
self.http_client.close() # this not imple
self.http_client = None
self.stopped = True
def dispatch_mutil_task(request_ip_port_params_list, uri, http_method):
http_client = AsyncHTTPClient()
_error_record_dict = {}
adminUser, adminPasswd = _retrieve_userName_passwd()
try:
_key_sets = set()
for (req_ip, req_port, params) in request_ip_port_params_list:
requesturl = "http://%s:%s%s" % (req_ip, req_port, uri)
logging.info('requesturi: %s' % requesturl)
logging.info('dispatch mutil task params :%s' % str(params))
request = HTTPRequest(url=requesturl, method=http_method, body=urllib.urlencode(params), \
connect_timeout=40, request_timeout=40, auth_username=adminUser, auth_password=adminPasswd)
callback_key = "%s_%s_%s" % (uri, req_ip, req_port)
_key_sets.add(callback_key)
http_client.fetch(request, callback=(yield Callback(callback_key)))
for callback_key in _key_sets:
response = yield Wait(callback_key)
if response.error:
return_result = False
error_record_msg = "remote access,the key:%s,error message:%s" % (
callback_key, response.error)
else:
return_result = response.body.strip()
if cmp('false', return_result) == 0:
callback_key_ip = callback_key.split("_")[-1]
_error_record_dict.setdefault(callback_key_ip,
error_record_msg)
if len(_error_record_dict) > 0:
raise CommonException('request occurs error! detail: %s' %
str(_error_record_dict))
else:
logging.info('request finished all!')
finally:
http_client.close()
async def get(self, pkg_url):
async def download_success(req, file_path: str, content: bytes):
async with AIOFile(file_path, 'wb') as afp:
await afp.write(content)
await afp.fsync()
req.write(content)
req.set_header('Content-Type', 'application/octet-stream')
await req.finish()
log.info("fetch package {}".format(pkg_url))
pkg_path = pkg_base_path / pkg_url
if not pkg_path.parent.exists():
pkg_path.parent.mkdir(parents=True)
http_client = AsyncHTTPClient()
pkg_url_custom = None
try:
results = re.search(re_suffix, pkg_url)
if results is None:
raise Exception("version not recognised {}".format(pkg_url))
pkg_id = pkg_url.split("/-/")[0]
pkg_url_custom = "{}/{}/download/{}{}".format(
pkg_base_custom, pkg_id, pkg_id, results.group())
r = await http_client.fetch(pkg_url_custom)
except Exception as e:
log.warning("fetch {} failed: {}".format(pkg_url_custom, e))
pkg_url_npm = "{}/{}".format(pkg_base_npm, pkg_url)
try:
r = await http_client.fetch(pkg_url_npm, request_timeout=300.0)
except Exception as e:
log.error("fetch {} failed: {}".format(pkg_url_npm, e))
else:
await download_success(self, str(pkg_path), r.body)
else:
await download_success(self, str(pkg_path), r.body)
finally:
http_client.close()
def post(self, product):
args = self.json_args
user_id = self.current_user['partner_id']
args['user_id'] = user_id
args['criteria']['product'] = product
if 'admin' not in self.current_user['roles']:
args['criteria']['user_id'] = user_id
body = json.dumps(args)
http_client = AsyncHTTPClient()
try:
downstream = self.application.config['downstream'][user_id]
iv = downstream['iv']
passphrase = downstream['pass']
aes = AES.new(passphrase, AES.MODE_CBC, iv)
b = aes.encrypt(padding(body))
encrypted = base64.b64encode(b).decode()
base_url = self.application.config['connection']['glados_hall']
url = base_url + '/export/request'
headers = {'User': user_id}
response = yield http_client.fetch(url, method='POST', headers=headers, body=encrypted)
resp = response.body.decode()
self.finish(resp)
except:
request_log.exception('EXPORT FAIL')
self.finish(json.dumps({'status': 'fail', 'msg': '导出异常，请稍后尝试'}))
finally:
http_client.close()
def fetch(self, url, method=None, headers=None, body=None, callback=None):
"""
执行请求
:param url:
:param method:
:param headers:
:param body:
:param callback: 回调函数的值为HttpResponse
:return:
"""
http_client = AsyncHTTPClient()
try:
request = HTTPRequest(url,
method=method,
headers=headers,
body=body)
http_client.fetch(request, callback)
except HTTPError as e:
gen_log.warning("HttpHelper fetch Error: " + str(e))
except Exception as e:
gen_log("HttpHelper fetch Error: " + str(e))
http_client.close()
def post_detect(self, user_id: str, application_id: str, session_id: str, locale: str, query: str, callback) -> str:
self.logger.debug(
"user_id=%s,application_id=%s,session_id=%s,locale=%s,query=%s",
user_id, application_id, session_id, locale, query
)
url = "%s?application_id=%s&session_id=%s&locale=%s&q=%s" % (
DETECT_URL,
application_id,
session_id,
locale,
url_escape(query)
# url_escape(json_encode(context))
)
if user_id is not None:
url += "&user_id=%s" % user_id
http_client = AsyncHTTPClient()
http_client.fetch(
HTTPRequest(url=url, method="POST", body=json_encode({})),
callback=callback
)
http_client.close()
def delete_favorite(self, handler: WebSocketHandler, user_id: ObjectId, product_id: ObjectId):
def delete_favorite_callback(response, handler):
self.favorites_cache.remove(user_id)
self.logger.debug("delete_favorite_completed")
self.logger.debug(
"user_id=%s,product_id=%s",
user_id, product_id
)
try:
url = "%s/%s/favorite/%s" % (USER_URL, user_id, product_id)
http_client = AsyncHTTPClient()
http_client.fetch(
HTTPRequest(url=url, method="DELETE"),
callback=lambda res: delete_favorite_callback(res, handler)
)
http_client.close()
except HTTPError as e:
self.logger.error("delete_favorite,url=%s", url)
raise
def post_update(self):
http_client = AsyncHTTPClient()
try:
base_url = self.application.config['connection']['repo']
url = base_url + '/api/user/update'
self.json_args['domain_id'] = self.current_user['domain_id']
body = json.dumps(self.json_args)
response = yield http_client.fetch(url, method='POST', body=body)
resp = response.body.decode()
self.finish(resp)
except ValueError as e:
status = 'fail'
msg = '代理商更新失败：' + str(e)
self.finish(json.dumps({'status': status, 'msg': msg}))
finally:
http_client.close()
def get_suggestion_items(self, user_id: str, application_id: str,
session_id: str, locale: str, suggestion_id: str,
page_size: int, offset: int, callback):
self.logger.debug(
"user_id=%s,application_id=%s,session_id=%s,locale=%s,"
"suggestion_id=%s,page_size=%s,offset=%s", user_id, application_id,
session_id, locale, suggestion_id, page_size, offset)
url = "%s/%s/items?session_id=%s&application_id=%s&locale=%s&page_size=%s&offset=%s" % (
SUGGEST_URL, suggestion_id, session_id, application_id, locale,
page_size, offset)
url += "&user_id=%s" % user_id if user_id is not None else ""
try:
http_client = AsyncHTTPClient()
http_client.fetch(HTTPRequest(url=url, method="GET"),
callback=callback)
http_client.close()
except HTTPError:
self.logger.error("url=%s", url)
raise
def aspire_ec_callback(OperSeq, mobile_list):
#body = CALLBACK.format(TransIDO, MobNum)
SuccNum = 0
SuccessTelList = ''
FailNum = 0
FailTelList = ''
# for mobile in mobile_list:
# SuccessTelList += SUCCESS_TEL.format(Mobile = mobile)
for index, mobile in enumerate(mobile_list):
if index % 2 == 0:
SuccNum += 1
SuccessTelList += SUCCESS_TEL.format(Mobile=mobile)
else:
FailNum += 1
FailTelList += FAIL_TEL.format(Mobile=mobile)
body = CALLBACK_SUCC.format(OperSeq=OperSeq,
SuccNum=SuccNum,
SuccessTelList=SuccessTelList,
FailNum=FailNum,
FailTelList=FailTelList)
http_client = AsyncHTTPClient()
try:
request_log.info('ASPIRE_EC CALLBACK\n%s', body)
http_client.fetch(
'http://192.168.1.152:8899/callback/cmcc3.do',
method='POST',
body=body,
headers={'Content-Type': 'application/xml;charset=UTF-8'})
except Exception:
request_log.exception('FAIL')
finally:
http_client.close()
def post_approval(self):
http_client = AsyncHTTPClient()
try:
user_name = self.json_args.get('name')
login = self.json_args.get('login')
request_id = self.json_args.get('request_id')
if user_name is None or user_name == '':
raise ValueError('用户名为空')
if login is None or login == '':
raise ValueError('登录名为空')
self.json_args['domain_id'] = self.current_user['domain_id']
body = json.dumps(self.json_args)
base_url = self.application.config['connection']['repo']
url = base_url + '/api/user/add'
response = yield http_client.fetch(url, method='POST', body=body)
resp = response.body.decode()
self.finish(resp)
if request_id and self.master.sismember('set:user:request',
request_id):
self.master.srem('set:user:request', request_id)
self.master.delete('request:%s' % request_id)
except Exception as e:
status = 'fail'
msg = '代理商添加失败：' + str(e)
self.finish(json.dumps({'status': status, 'msg': msg}))
finally:
http_client.close()
def _check_heapster_available(self):
if "HEAPSTER_SERVICE_HOST" in os.environ:
heapster_endpoint = os.getenv("HEAPSTER_SERVICE_HOST")
heapster_port = os.getenv("HEAPSTER_SERVICE_PORT")
url = "http://%s:%s/api/v1/model" % (heapster_endpoint, heapster_port)
client = AsyncHTTPClient(force_instance=True)
try:
result = yield client.fetch(url, method="GET", raise_error=False)
if not result.error:
self.heapster_base_url = url
finally:
client.close()
else:
try:
yield self.http_client.get("/api/v1/proxy/namespaces/kube-system/services/heapster/api/v1/model")
self.heapster_base_url = "%s%s" % (
self.http_client._base_url,
"/api/v1/proxy/namespaces/kube-system/services/heapster/api/v1/model")
except HTTPError as http_error:
logging.exception(http_error)
logging.debug("Cannot access Heaptser API through proxy API")
async def get(self):
http_client = AsyncHTTPClient()
lang = self.get_argument('lang', 'en')
try:
self.set_header("Content-Type", "text/xml")
url = (
"https://wmts.geo.admin.ch/EPSG/2056/"
"1.0.0/WMTSCapabilities.xml?lang={}"
).format(lang)
print(f"Fetching WMTS GetCapability: {url}")
response = await http_client.fetch(url)
print("Done.")
self.write(response.body)
except HTTPError as e:
print("Error: " + str(e))
except Exception as e:
# Other errors are possible, such as IOError.
print("Error: " + str(e))
http_client.close()
def post_suggest(self, user_id: str, application_id: str, session_id: str,
locale: str, context: dict, callback) -> str:
self.logger.debug(
"user_id=%s,application_id=%s,session_id=%s,locale=%s,"
"context=%s", user_id, application_id, session_id, locale, context)
url = "%s?session_id=%s&application_id=%s&locale=%s" % (
SUGGEST_URL, session_id, application_id, locale)
url += "&user_id=%s" % user_id if user_id is not None else ""
try:
request_body = {"context": context}
http_client = AsyncHTTPClient()
http_client.fetch(HTTPRequest(url=url,
body=dumps(request_body),
method="POST"),
callback=callback)
http_client.close()
except HTTPError:
self.logger.error("url=%s", url)
raise
def xiamen_zyt_callback(order_id, result):
if ';' in result:
result = result.split(';')[0]
body = json.dumps({
"channelNo": "yiqichong001",
"orderno": order_id,
"responsetime": "20160116073449",
"resultcode": result
})
http_client = AsyncHTTPClient()
try:
request_log.info('XIAMEN_ZYT CALLBACK\n%s', body)
http_client.fetch('http://localhost:8899/callback/xiamen_zyt',
method='POST',
body=body)
except Exception:
request_log.exception('FAIL')
finally:
http_client.close()
class AsyncHTTPTestCase(AsyncTestCase):
def setUp(self):
self.stopped = False
self.running = False
self.failure = None
self.stop_args = None
super(AsyncHTTPTestCase, self).setUp()
self.port = None
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
if hasattr(self, 'get_app'):
self.app = self.get_app()
self.http_server = HTTPServer(self.app, io_loop=self.io_loop,
**self.get_httpserver_options())
self.http_server.listen(self.get_http_port())
def fetch(self, path, **kwargs):
self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
return self.wait()
def get_httpserver_options(self):
return {}
def get_http_port(self):
if self.port is None:
self.port = get_unused_port()
return self.port
def get_url(self, path):
return 'http://localhost:%s%s' % (self.get_http_port(), path)
def tearDown(self):
self.http_server.stop()
self.http_client.close()
super(AsyncHTTPTestCase, self).tearDown()
def trafficweb_callback(order_id, result):
if ';' in result:
result = result.split(';')[0]
data = {
"order_id": "2016012710232187985",
"client_order_id": order_id,
"status": result,
"desc": "\\u8ba2\\u8d2d\\u6210\\u529f"
}
body = json.dumps(data)
http_client = AsyncHTTPClient()
try:
request_log.info('TRAFFICWEB CALLBACK\n%s', body)
http_client.fetch('http://localhost:8899/callback/bitefeng',
method='POST',
body=body)
except Exception:
request_log.exception('FAIL')
finally:
http_client.close()
def ibumobile_callback(order_id, result):
desc = ''
if ';' in result:
result, desc = result.split(';')
body = {
"orderNo": order_id,
"resCode": result,
"redMsg": desc,
}
body = urlencode(body)
http_client = AsyncHTTPClient()
try:
request_log.info('IBUMOBILE CALLBACK\n%s', body)
http_client.fetch('http://localhost:8899/callback/ibumobile',
method='POST',
body=body)
except Exception:
request_log.exception('FAIL')
finally:
http_client.close()
class AsyncHTTPTestCase(AsyncTestCase):
'''A test case that starts up an HTTP server.
Subclasses must override get_app(), which returns the
tornado.web.Application (or other HTTPServer callback) to be tested.
Tests will typically use the provided self.http_client to fetch
URLs from this server.
Example:
class MyHTTPTest(AsyncHTTPTestCase):
def get_app(self):
return Application([('/', MyHandler)...])
def test_homepage(self):
# The following two lines are equivalent to
# response = self.fetch('/')
# but are shown in full here to demonstrate explicit use
# of self.stop and self.wait.
self.http_client.fetch(self.get_url('/'), self.stop)
response = self.wait()
# test contents of response
'''
def setUp(self):
super(AsyncHTTPTestCase, self).setUp()
self.__port = None
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
self._app = self.get_app()
self.http_server = HTTPServer(self._app,
io_loop=self.io_loop,
**self.get_httpserver_options())
self.http_server.listen(self.get_http_port())
def get_app(self):
"""Should be overridden by subclasses to return a
tornado.web.Application or other HTTPServer callback.
"""
raise NotImplementedError()
def fetch(self, path, **kwargs):
"""Convenience method to synchronously fetch a url.
The given path will be appended to the local server's host and port.
Any additional kwargs will be passed directly to
AsyncHTTPClient.fetch (and so could be used to pass method="POST",
body="...", etc).
"""
self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
return self.wait()
def get_httpserver_options(self):
"""May be overridden by subclasses to return additional
keyword arguments for HTTPServer.
"""
return {}
def get_http_port(self):
"""Returns the port used by the HTTPServer.
A new port is chosen for each test.
"""
if self.__port is None:
self.__port = get_unused_port()
return self.__port
def get_url(self, path):
"""Returns an absolute url for the given path on the test server."""
return 'http://localhost:%s%s' % (self.get_http_port(), path)
def tearDown(self):
self.http_server.stop()
self.http_client.close()
super(AsyncHTTPTestCase, self).tearDown()
def up_wo(handler, partner):
product_info = handler.slave.hmget(
'private:wo:%s:%d' % (handler.area, handler.price),
['product_id', 'price'])
product_id = product_info[0]
price = product_info[1]
activity_id = partner['activity_id']
handler.up_req_time = time.localtime()
tsp = time.strftime("%Y%m%d%H%M%S", handler.up_req_time)
order_id = tsp + '000' + handler.order_id[-3:]
handler.up_order_id = order_id
# key+orderNo+phoneNum+productId+ activityId+realPrice)
token = wo_key(partner['key'] + order_id + handler.mobile + product_id +
activity_id + price)
query = (
'orderNo={order_id}&productId={product_id}&activityId={activity_id}'
'&phoneNum={mobile}&realPrice={price}&token={token}').format(
order_id=order_id,
product_id=product_id,
activity_id=activity_id,
mobile=handler.mobile,
price=price,
token=token)
url = partner['url.order'] + '?' + query
# print(handler.order_id + ":" + body)
request_log.info('CALL_REQ %s', query, extra={'orderid': handler.order_id})
# call & wait
response = None
result = 99999
http_client = AsyncHTTPClient()
try:
response = yield http_client.fetch(url, method='GET')
except HTTPError as http_error:
request_log.error('CALL UPSTREAM FAIL %s',
http_error,
extra={'orderid': handler.order_id})
result = 60000 + http_error.code
except Exception as e:
request_log.error('CALL UPSTREAM FAIL %s',
e,
extra={'orderid': handler.order_id})
finally:
http_client.close()
handler.up_resp_time = time.localtime() # <--------------
if response and response.code == 200:
body = response.body.decode('utf8')
request_log.info('CALL_RESP %s',
body,
extra={'orderid': handler.order_id})
try:
resp = json.loads(body)
status = resp['status']
result = RESULT_MAPPING.get(status, status)
handler.up_result = result
except Exception as e:
result = 99999
request_log.error('PARSE UPSTREAM %s',
e,
extra={'orderid': handler.order_id})
if handler.up_result is None:
handler.up_result = result
if result in [0, 60599]:
for _ in range(5):
result = yield query_order(order_id, partner)
if result in [1, 9]:
break
# wait
yield tornado.gen.Task(IOLoop.instance().add_timeout,
time.time() + 120)
if result == 1:
yield handler.callback('1')
elif result == 9:
pass
return result
class CommandHelper:
def __init__(self, config: ConfigHelper) -> None:
self.server = config.get_server()
self.debug_enabled = config.getboolean('enable_repo_debug', False)
if self.debug_enabled:
logging.warning("UPDATE MANAGER: REPO DEBUG ENABLED")
shell_cmd: SCMDComp = self.server.lookup_component('shell_command')
self.scmd_error = shell_cmd.error
self.build_shell_command = shell_cmd.build_shell_command
self.pkg_updater: Optional[PackageDeploy] = None
AsyncHTTPClient.configure(None, defaults=dict(user_agent="Moonraker"))
self.http_client = AsyncHTTPClient()
self.github_request_cache: Dict[str, CachedGithubResponse] = {}
# GitHub API Rate Limit Tracking
self.gh_rate_limit: Optional[int] = None
self.gh_limit_remaining: Optional[int] = None
self.gh_limit_reset_time: Optional[float] = None
# Update In Progress Tracking
self.cur_update_app: Optional[str] = None
self.cur_update_id: Optional[int] = None
self.full_complete: bool = False
def get_server(self) -> Server:
return self.server
def is_debug_enabled(self) -> bool:
return self.debug_enabled
def set_update_info(self,
app: str,
uid: int,
full_complete: bool = True
) -> None:
self.cur_update_app = app
self.cur_update_id = uid
self.full_complete = full_complete
def set_full_complete(self, complete: bool = False):
self.full_complete = complete
def clear_update_info(self) -> None:
self.cur_update_app = self.cur_update_id = None
self.full_complete = False
def is_app_updating(self, app_name: str) -> bool:
return self.cur_update_app == app_name
def is_update_busy(self) -> bool:
return self.cur_update_app is not None
def set_package_updater(self, updater: PackageDeploy) -> None:
self.pkg_updater = updater
def get_rate_limit_stats(self) -> Dict[str, Any]:
return {
'github_rate_limit': self.gh_rate_limit,
'github_requests_remaining': self.gh_limit_remaining,
'github_limit_reset_time': self.gh_limit_reset_time,
}
async def init_api_rate_limit(self) -> None:
url = "https://api.github.com/rate_limit"
while 1:
try:
resp = await self.github_api_request(url, is_init=True)
assert isinstance(resp, dict)
core = resp['resources']['core']
self.gh_rate_limit = core['limit']
self.gh_limit_remaining = core['remaining']
self.gh_limit_reset_time = core['reset']
except Exception:
logging.exception("Error Initializing GitHub API Rate Limit")
await asyncio.sleep(30.)
else:
reset_time = time.ctime(self.gh_limit_reset_time)
logging.info(
"GitHub API Rate Limit Initialized\n"
f"Rate Limit: {self.gh_rate_limit}\n"
f"Rate Limit Remaining: {self.gh_limit_remaining}\n"
f"Rate Limit Reset Time: {reset_time}, "
f"Seconds Since Epoch: {self.gh_limit_reset_time}")
break
async def run_cmd(self,
cmd: str,
timeout: float = 20.,
notify: bool = False,
retries: int = 1,
env: Optional[Dict[str, str]] = None,
cwd: Optional[str] = None,
sig_idx: int = 1
) -> None:
cb = self.notify_update_response if notify else None
scmd = self.build_shell_command(cmd, callback=cb, env=env, cwd=cwd)
while retries:
if await scmd.run(timeout=timeout, sig_idx=sig_idx):
break
retries -= 1
if not retries:
raise self.server.error("Shell Command Error")
async def run_cmd_with_response(self,
cmd: str,
timeout: float = 20.,
retries: int = 5,
env: Optional[Dict[str, str]] = None,
cwd: Optional[str] = None,
sig_idx: int = 1
) -> str:
scmd = self.build_shell_command(cmd, None, env=env, cwd=cwd)
result = await scmd.run_with_response(timeout, retries,
sig_idx=sig_idx)
return result
async def github_api_request(self,
url: str,
is_init: Optional[bool] = False
) -> JsonType:
if self.gh_limit_remaining == 0:
curtime = time.time()
assert self.gh_limit_reset_time is not None
if curtime < self.gh_limit_reset_time:
raise self.server.error(
f"GitHub Rate Limit Reached\nRequest: {url}\n"
f"Limit Reset Time: {time.ctime(self.gh_limit_remaining)}")
if url in self.github_request_cache:
cached_request = self.github_request_cache[url]
etag: Optional[str] = cached_request.get_etag()
else:
cached_request = CachedGithubResponse()
etag = None
self.github_request_cache[url] = cached_request
headers = {"Accept": "application/vnd.github.v3+json"}
if etag is not None:
headers['If-None-Match'] = etag
retries = 5
while retries:
try:
fut = self.http_client.fetch(
url, headers=headers, connect_timeout=5.,
request_timeout=5., raise_error=False)
resp: HTTPResponse
resp = await asyncio.wait_for(fut, 10.)
except Exception:
retries -= 1
if retries > 0:
logging.exception(
f"Error Processing GitHub API request: {url}")
await asyncio.sleep(1.)
continue
etag = resp.headers.get('etag', None)
if etag is not None:
if etag[:2] == "W/":
etag = etag[2:]
logging.info(
"GitHub API Request Processed\n"
f"URL: {url}\n"
f"Response Code: {resp.code}\n"
f"Response Reason: {resp.reason}\n"
f"ETag: {etag}")
if resp.code == 403:
raise self.server.error(
f"Forbidden GitHub Request: {resp.reason}")
elif resp.code == 304:
logging.info(f"Github Request not Modified: {url}")
return cached_request.get_cached_result()
if resp.code != 200:
retries -= 1
if not retries:
raise self.server.error(
f"Github Request failed: {resp.code} {resp.reason}")
logging.info(
f"Github request error, {retries} retries remaining")
await asyncio.sleep(1.)
continue
# Update rate limit on return success
if 'X-Ratelimit-Limit' in resp.headers and not is_init:
self.gh_rate_limit = int(resp.headers['X-Ratelimit-Limit'])
self.gh_limit_remaining = int(
resp.headers['X-Ratelimit-Remaining'])
self.gh_limit_reset_time = float(
resp.headers['X-Ratelimit-Reset'])
decoded = json.loads(resp.body)
if etag is not None:
cached_request.update_result(etag, decoded)
return decoded
raise self.server.error(
f"Retries exceeded for GitHub API request: {url}")
async def http_download_request(self,
url: str,
content_type: str,
timeout: float = 180.
) -> bytes:
retries = 5
while retries:
try:
fut = self.http_client.fetch(
url, headers={"Accept": content_type},
connect_timeout=5., request_timeout=timeout)
resp: HTTPResponse
resp = await asyncio.wait_for(fut, timeout + 10.)
except Exception:
retries -= 1
logging.exception("Error Processing Download")
if not retries:
raise
await asyncio.sleep(1.)
continue
return resp.body
raise self.server.error(
f"Retries exceeded for GitHub API request: {url}")
async def streaming_download_request(self,
url: str,
dest: Union[str, pathlib.Path],
content_type: str,
size: int,
timeout: float = 180.
) -> None:
if isinstance(dest, str):
dest = pathlib.Path(dest)
retries = 5
while retries:
dl = StreamingDownload(self, dest, size)
try:
fut = self.http_client.fetch(
url, headers={"Accept": content_type},
connect_timeout=5., request_timeout=timeout,
streaming_callback=dl.on_chunk_recd)
resp: HTTPResponse
resp = await asyncio.wait_for(fut, timeout + 10.)
except Exception:
retries -= 1
logging.exception("Error Processing Download")
if not retries:
raise
await asyncio.sleep(1.)
continue
finally:
await dl.close()
if resp.code < 400:
return
def notify_update_response(self,
resp: Union[str, bytes],
is_complete: bool = False
) -> None:
if self.cur_update_app is None:
return
resp = resp.strip()
if isinstance(resp, bytes):
resp = resp.decode()
done = is_complete and self.full_complete
notification = {
'message': resp,
'application': self.cur_update_app,
'proc_id': self.cur_update_id,
'complete': done}
self.server.send_event(
"update_manager:update_response", notification)
async def install_packages(self,
package_list: List[str],
**kwargs
) -> None:
if self.pkg_updater is None:
return
await self.pkg_updater.install_packages(package_list, **kwargs)
def close(self) -> None:
self.http_client.close()
class UpdateManager:
def __init__(self, config):
self.server = config.get_server()
self.config = config
self.config.read_supplemental_config(SUPPLEMENTAL_CFG_PATH)
self.repo_debug = config.getboolean('enable_repo_debug', False)
auto_refresh_enabled = config.getboolean('enable_auto_refresh', False)
self.distro = config.get('distro', "debian").lower()
if self.distro not in SUPPORTED_DISTROS:
raise config.error(f"Unsupported distro: {self.distro}")
if self.repo_debug:
logging.warn("UPDATE MANAGER: REPO DEBUG ENABLED")
env = sys.executable
mooncfg = self.config[f"update_manager static {self.distro} moonraker"]
self.updaters = {
"system": PackageUpdater(self),
"moonraker": GitUpdater(self, mooncfg, MOONRAKER_PATH, env)
}
self.current_update = None
# TODO: Check for client config in [update_manager]. This is
# deprecated and will be removed.
client_repo = config.get("client_repo", None)
if client_repo is not None:
client_path = config.get("client_path")
name = client_repo.split("/")[-1]
self.updaters[name] = WebUpdater(self, {
'repo': client_repo,
'path': client_path
})
client_sections = self.config.get_prefix_sections(
"update_manager client")
for section in client_sections:
cfg = self.config[section]
name = section.split()[-1]
if name in self.updaters:
raise config.error("Client repo named %s already added" %
(name, ))
client_type = cfg.get("type")
if client_type == "git_repo":
self.updaters[name] = GitUpdater(self, cfg)
elif client_type == "web":
self.updaters[name] = WebUpdater(self, cfg)
else:
raise config.error("Invalid type '%s' for section [%s]" %
(client_type, section))
# GitHub API Rate Limit Tracking
self.gh_rate_limit = None
self.gh_limit_remaining = None
self.gh_limit_reset_time = None
self.gh_init_evt = Event()
self.cmd_request_lock = Lock()
self.is_refreshing = False
# Auto Status Refresh
self.last_auto_update_time = 0
self.refresh_cb = None
if auto_refresh_enabled:
self.refresh_cb = PeriodicCallback(self._handle_auto_refresh,
UPDATE_REFRESH_INTERVAL_MS)
self.refresh_cb.start()
AsyncHTTPClient.configure(None, defaults=dict(user_agent="Moonraker"))
self.http_client = AsyncHTTPClient()
self.server.register_endpoint("/machine/update/moonraker", ["POST"],
self._handle_update_request)
self.server.register_endpoint("/machine/update/klipper", ["POST"],
self._handle_update_request)
self.server.register_endpoint("/machine/update/system", ["POST"],
self._handle_update_request)
self.server.register_endpoint("/machine/update/client", ["POST"],
self._handle_update_request)
self.server.register_endpoint("/machine/update/status", ["GET"],
self._handle_status_request)
# Register Ready Event
self.server.register_event_handler("server:klippy_identified",
self._set_klipper_repo)
# Initialize GitHub API Rate Limits and configured updaters
IOLoop.current().spawn_callback(self._initalize_updaters,
list(self.updaters.values()))
async def _initalize_updaters(self, initial_updaters):
self.is_refreshing = True
await self._init_api_rate_limit()
for updater in initial_updaters:
if isinstance(updater, PackageUpdater):
ret = updater.refresh(False)
else:
ret = updater.refresh()
if asyncio.iscoroutine(ret):
await ret
self.is_refreshing = False
async def _set_klipper_repo(self):
kinfo = self.server.get_klippy_info()
if not kinfo:
logging.info("No valid klippy info received")
return
kpath = kinfo['klipper_path']
env = kinfo['python_path']
kupdater = self.updaters.get('klipper', None)
if kupdater is not None and kupdater.repo_path == kpath and \
kupdater.env == env:
# Current Klipper Updater is valid
return
kcfg = self.config[f"update_manager static {self.distro} klipper"]
self.updaters['klipper'] = GitUpdater(self, kcfg, kpath, env)
await self.updaters['klipper'].refresh()
async def _check_klippy_printing(self):
klippy_apis = self.server.lookup_plugin('klippy_apis')
result = await klippy_apis.query_objects({'print_stats': None},
default={})
pstate = result.get('print_stats', {}).get('state', "")
return pstate.lower() == "printing"
async def _handle_auto_refresh(self):
if await self._check_klippy_printing():
# Don't Refresh during a print
logging.info("Klippy is printing, auto refresh aborted")
return
cur_time = time.time()
cur_hour = time.localtime(cur_time).tm_hour
time_diff = cur_time - self.last_auto_update_time
# Update packages if it has been more than 12 hours
# and the local time is between 12AM and 5AM
if time_diff < MIN_REFRESH_TIME or cur_hour >= MAX_PKG_UPDATE_HOUR:
# Not within the update time window
return
self.last_auto_update_time = cur_time
vinfo = {}
need_refresh_all = not self.is_refreshing
async with self.cmd_request_lock:
self.is_refreshing = True
try:
for name, updater in list(self.updaters.items()):
if need_refresh_all:
ret = updater.refresh()
if asyncio.iscoroutine(ret):
await ret
if hasattr(updater, "get_update_status"):
vinfo[name] = updater.get_update_status()
except Exception:
logging.exception("Unable to Refresh Status")
return
finally:
self.is_refreshing = False
uinfo = {
'version_info': vinfo,
'github_rate_limit': self.gh_rate_limit,
'github_requests_remaining': self.gh_limit_remaining,
'github_limit_reset_time': self.gh_limit_reset_time,
'busy': self.current_update is not None
}
self.server.send_event("update_manager:update_refreshed", uinfo)
async def _handle_update_request(self, web_request):
if await self._check_klippy_printing():
raise self.server.error("Update Refused: Klippy is printing")
app = web_request.get_endpoint().split("/")[-1]
if app == "client":
app = web_request.get('name')
inc_deps = web_request.get_boolean('include_deps', False)
if self.current_update is not None and \
self.current_update[0] == app:
return f"Object {app} is currently being updated"
updater = self.updaters.get(app, None)
if updater is None:
raise self.server.error(f"Updater {app} not available")
async with self.cmd_request_lock:
self.current_update = (app, id(web_request))
try:
await updater.update(inc_deps)
except Exception as e:
self.notify_update_response(f"Error updating {app}")
self.notify_update_response(str(e), is_complete=True)
raise
finally:
self.current_update = None
return "ok"
async def _handle_status_request(self, web_request):
check_refresh = web_request.get_boolean('refresh', False)
# Don't refresh if a print is currently in progress or
# if an update is in progress. Just return the current
# state
if self.current_update is not None or \
await self._check_klippy_printing():
check_refresh = False
need_refresh = False
if check_refresh:
# If there is an outstanding request processing a
# refresh, we don't need to do it again.
need_refresh = not self.is_refreshing
await self.cmd_request_lock.acquire()
self.is_refreshing = True
vinfo = {}
try:
for name, updater in list(self.updaters.items()):
await updater.check_initialized(120.)
if need_refresh:
ret = updater.refresh()
if asyncio.iscoroutine(ret):
await ret
if hasattr(updater, "get_update_status"):
vinfo[name] = updater.get_update_status()
except Exception:
raise
finally:
if check_refresh:
self.is_refreshing = False
self.cmd_request_lock.release()
return {
'version_info': vinfo,
'github_rate_limit': self.gh_rate_limit,
'github_requests_remaining': self.gh_limit_remaining,
'github_limit_reset_time': self.gh_limit_reset_time,
'busy': self.current_update is not None
}
async def execute_cmd(self, cmd, timeout=10., notify=False, retries=1):
shell_command = self.server.lookup_plugin('shell_command')
cb = self.notify_update_response if notify else None
scmd = shell_command.build_shell_command(cmd, callback=cb)
while retries:
if await scmd.run(timeout=timeout, verbose=notify):
break
retries -= 1
if not retries:
raise self.server.error("Shell Command Error")
async def execute_cmd_with_response(self, cmd, timeout=10.):
shell_command = self.server.lookup_plugin('shell_command')
scmd = shell_command.build_shell_command(cmd, None)
result = await scmd.run_with_response(timeout, retries=5)
if result is None:
raise self.server.error(f"Error Running Command: {cmd}")
return result
async def _init_api_rate_limit(self):
url = "https://api.github.com/rate_limit"
while 1:
try:
resp = await self.github_api_request(url, is_init=True)
core = resp['resources']['core']
self.gh_rate_limit = core['limit']
self.gh_limit_remaining = core['remaining']
self.gh_limit_reset_time = core['reset']
except Exception:
logging.exception("Error Initializing GitHub API Rate Limit")
await tornado.gen.sleep(30.)
else:
reset_time = time.ctime(self.gh_limit_reset_time)
logging.info(
"GitHub API Rate Limit Initialized\n"
f"Rate Limit: {self.gh_rate_limit}\n"
f"Rate Limit Remaining: {self.gh_limit_remaining}\n"
f"Rate Limit Reset Time: {reset_time}, "
f"Seconds Since Epoch: {self.gh_limit_reset_time}")
break
self.gh_init_evt.set()
async def github_api_request(self, url, etag=None, is_init=False):
if not is_init:
timeout = time.time() + 30.
try:
await self.gh_init_evt.wait(timeout)
except Exception:
raise self.server.error("Timeout while waiting for GitHub "
"API Rate Limit initialization")
if self.gh_limit_remaining == 0:
curtime = time.time()
if curtime < self.gh_limit_reset_time:
raise self.server.error(
f"GitHub Rate Limit Reached\nRequest: {url}\n"
f"Limit Reset Time: {time.ctime(self.gh_limit_remaining)}")
headers = {"Accept": "application/vnd.github.v3+json"}
if etag is not None:
headers['If-None-Match'] = etag
retries = 5
while retries:
try:
timeout = time.time() + 10.
fut = self.http_client.fetch(url,
headers=headers,
connect_timeout=5.,
request_timeout=5.,
raise_error=False)
resp = await tornado.gen.with_timeout(timeout, fut)
except Exception:
retries -= 1
msg = f"Error Processing GitHub API request: {url}"
if not retries:
raise self.server.error(msg)
logging.exception(msg)
await tornado.gen.sleep(1.)
continue
etag = resp.headers.get('etag', None)
if etag is not None:
if etag[:2] == "W/":
etag = etag[2:]
logging.info("GitHub API Request Processed\n"
f"URL: {url}\n"
f"Response Code: {resp.code}\n"
f"Response Reason: {resp.reason}\n"
f"ETag: {etag}")
if resp.code == 403:
raise self.server.error(
f"Forbidden GitHub Request: {resp.reason}")
elif resp.code == 304:
logging.info(f"Github Request not Modified: {url}")
return None
if resp.code != 200:
retries -= 1
if not retries:
raise self.server.error(
f"Github Request failed: {resp.code} {resp.reason}")
logging.info(
f"Github request error, {retries} retries remaining")
await tornado.gen.sleep(1.)
continue
# Update rate limit on return success
if 'X-Ratelimit-Limit' in resp.headers and not is_init:
self.gh_rate_limit = int(resp.headers['X-Ratelimit-Limit'])
self.gh_limit_remaining = int(
resp.headers['X-Ratelimit-Remaining'])
self.gh_limit_reset_time = float(
resp.headers['X-Ratelimit-Reset'])
decoded = json.loads(resp.body)
decoded['etag'] = etag
return decoded
async def http_download_request(self, url):
retries = 5
while retries:
try:
timeout = time.time() + 130.
fut = self.http_client.fetch(
url,
headers={"Accept": "application/zip"},
connect_timeout=5.,
request_timeout=120.)
resp = await tornado.gen.with_timeout(timeout, fut)
except Exception:
retries -= 1
logging.exception("Error Processing Download")
if not retries:
raise
await tornado.gen.sleep(1.)
continue
return resp.body
def notify_update_response(self, resp, is_complete=False):
resp = resp.strip()
if isinstance(resp, bytes):
resp = resp.decode()
notification = {
'message': resp,
'application': None,
'proc_id': None,
'complete': is_complete
}
if self.current_update is not None:
notification['application'] = self.current_update[0]
notification['proc_id'] = self.current_update[1]
self.server.send_event("update_manager:update_response", notification)
def close(self):
self.http_client.close()
if self.refresh_cb is not None:
self.refresh_cb.stop()
class TestMockHTTP(AsyncTestCase):
def setUp(self):
super(TestMockHTTP, self).setUp()
self.mock = MockHTTP()
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
def tearDown(self):
self.mock.shutdown()
self.http_client.close()
super(TestMockHTTP, self).tearDown()
def fetch(self, path, **kwargs):
"""Convenience method to synchronously fetch a url.
The given path will be appended to the local server's host and
port. Any additional kwargs will be passed directly to
`.AsyncHTTPClient.fetch` (and so could be used to pass
``method="POST"``, ``body="..."``, etc).
"""
self.http_client.fetch(
HTTPRequest(url=self.mock.get_url(path), **kwargs),
callback=self.stop)
return self.wait()
def test_get_request(self):
"""Tests a get request that expects nothing to return but an 200."""
self.mock.expects(method=GET, path='/index.html')
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.code, 200)
self.assert_(self.mock.verify())
def test_get_request_wrong_url(self):
"""Tests a get request that expects a different URL."""
self.mock.expects(method=GET, path='/index.html')
resp = self.fetch(method="GET", path="notindex.html")
self.assertEqual(resp.code, 404)
self.assertRaises(UnexpectedURLException, self.mock.verify)
def test_get_with_code(self):
self.mock.expects(method=GET, path='/index.html').will(http_code=500)
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.code, 500, 'Expected 500 response.')
self.assert_(self.mock.verify())
def test_get_with_body(self):
"""Tests a get request that returns a different URL."""
test_body = 'Test response.'
self.mock.expects(method=GET, path='/index.html').will(body=test_body)
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.body, test_body)
self.assert_(self.mock.verify())
def test_get_with_header(self):
"""Tests a get request that includes a custom header."""
test_header_name = 'Content-Type'
test_header_contents = 'text/html'
self.mock.expects(method=GET, path='/index.html').will(
headers={test_header_name: test_header_contents})
resp = self.fetch(method="GET", path="index.html")
self.assertTrue(resp.headers[test_header_name].startswith(test_header_contents))
self.assert_(self.mock.verify())
def test_multiple_get(self):
"""Test getting a URL twice."""
self.mock.expects(method=GET, path='/index.html')
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.code, 200)
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.code, 200)
self.assert_(self.mock.verify())
def test_never_get(self):
"""Test a URL that has a 'never' times on it."""
self.mock.expects(method=GET, path='/index.html', times=never)
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.code, 404)
self.assertRaises(UnexpectedURLException, self.mock.verify)
def test_get_once_got_twice(self):
"""Test getting a URL twice that expects to be retrieved once only."""
self.mock.expects(method=GET, path='/index.html', times=once)
resp = self.fetch(method="GET", path="index.html")
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.code, 404)
self.assertRaises(AlreadyRetrievedURLException, self.mock.verify)
def test_get_once_got_never(self):
"""Test never getting a URL that expects to be retrieved once only."""
self.mock.expects(method=GET, path='/index.html', times=once)
self.assertRaises(UnretrievedURLException, self.mock.verify)
def test_get_5times_got_never(self):
"""Test never getting a URL that expects to be retrieved 5 times only."""
self.mock.expects(method=GET, path='/index.html', times=5)
self.assertRaises(UnretrievedURLException, self.mock.verify)
def test_get_5times_got_5times(self):
"""Test getting 5 times a URL that expects to be retrieved 5 times"""
self.mock.expects(method=GET, path='/index.html', times=5)
call = lambda: self.fetch('/index.html')
[ call() for x in xrange(5) ]
self.mock.verify()
def test_get_at_least_once_got_twice(self):
"""Test getting a URL twice that expects to be retrieved at least once."""
self.mock.expects(method=GET, path='/index.html', times=at_least_once)
resp = self.fetch(method="GET", path="index.html")
self.assertEqual(resp.code, 200)
self.assert_(self.mock.verify())
def test_get_at_least_once_got_never(self):
"""Test never getting a URL that expects to be retrieved at least once."""
self.mock.expects(method=GET, path='/index.html', times=at_least_once)
self.assertRaises(UnretrievedURLException, self.mock.verify)
def test_get_after(self):
"""Test two URLs that expect to be retrieved in order."""
test_body = 'Test POST body.\r\n'
headers = {'content-type': 'text/plain'}
self.mock.expects(method=GET, path='/index.html', name='url #1')
self.mock.expects(method=POST, path='/index.html', after='url #1', headers=headers, body=test_body)
self.fetch(method="GET", path="index.html")
self.fetch(method="POST", path="index.html", body = test_body, headers=headers)
self.assert_(self.mock.verify())
def test_get_after_wrong_order(self):
"""Test two URLs that expect to be retrieved in order, but aren't."""
test_body = 'Test POST body.\r\n'
headers = {'content-type': 'text/plain'}
self.mock.expects(method=GET, path='/index.html', name='url #1')
self.mock.expects(method=POST, path='/index.html', after='url #1', body=test_body, headers=headers)
resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = headers)
self.assertEqual(resp.code, 404)
self.assertRaises(URLOrderingException, self.mock.verify)
def test_post(self):
"""Tests a POST request."""
test_body = 'Test POST body.\r\n'
headers = {'content-type': 'text/plain'}
self.mock.expects(method=POST, path='/index.html', body=test_body, headers=headers).will(http_code=201)
resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = headers)
self.assertEqual(resp.code, 201)
self.assert_(self.mock.verify())
def test_post_bad_body(self):
"""Tests a POST request that sends the wrong body data."""
test_body = 'Test POST body.\r\n'
expected_body = 'Expected POST body.\r\n'
headers = {'content-type': 'text/plain'}
self.mock.expects(method=POST, path='/index.html', body=expected_body)
resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = headers)
self.assertEqual(resp.code, 404)
self.assertRaises(WrongBodyException, self.mock.verify)
def test_post_header(self):
"""Tests a POST request with some custom headers."""
test_body = 'Test POST body.\r\n'
test_headers = {'content-type': 'application/atom+xml; type=entry',
'content-length': str(len(test_body)),
'Slug': 'ooze',}
self.mock.expects(method=POST, path='/index.html',
body=test_body, headers=test_headers).will(http_code=201)
resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers)
self.assertEqual(resp.code, 201)
self.assert_(self.mock.verify())
def test_post_unexpected_header(self):
"""Tests a POST request with an unexpected header."""
test_body = 'Test POST body.\r\n'
test_headers = {'content-type': 'application/atom+xml; type=entry',
'content-length': str(len(test_body)),
'Slug': 'ooze',}
expected_headers = {'content-type': 'application/atom+xml; type=entry',
'content-length': str(len(test_body)),}
self.mock.expects(method=POST, path='/index.html',
body=test_body, headers=expected_headers)
resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers)
self.assertEqual(resp.code, 200)
self.assert_(self.mock.verify())
def test_post_missing_header(self):
"""Tests a POST request without an expected header."""
test_body = 'Test POST body.\r\n'
test_headers = {'content-type': 'application/atom+xml; type=entry',
'content-length': str(len(test_body)),}
expected_headers = {'content-type': 'application/atom+xml; type=entry',
'content-length': str(len(test_body)),
'Slug': 'ooze',}
self.mock.expects(method=POST, path='/index.html',
body=test_body, headers=expected_headers)
resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers)
self.assertEqual(resp.code, 404)
self.assertRaises(WrongHeaderException, self.mock.verify)
def test_post_unexpected_header_value(self):
"""Tests a POST request with an unexpected header value."""
test_body = 'Test POST body.\r\n'
test_headers = {'content-type': 'application/atom+xml; type=entry',
'content-length': str(len(test_body)),
'Slug': 'ooze',}
expected_headers = {'content-type': 'application/atom+xml; type=entry',
'content-length': str(len(test_body)),
'Slug': 'slime',}
self.mock.expects(method=POST, path='/index.html',
body=test_body, headers=expected_headers)
resp = self.fetch('/index.html', method = 'POST', body = test_body, headers = test_headers)
self.assertEqual(resp.code, 404)
self.assertRaises(WrongHeaderValueException, self.mock.verify)
class TestClient(object):
def __init__(self, app, raise_handler_exc=False):
# TODO add `host` kwarg
if app.is_running:
raise RuntimeError('You should not instance TestClient'
'when applicaion is running')
self.raise_handler_exc = raise_handler_exc
# start::tornado.testing.AsyncTestCase
self.__stopped = False
self.__running = False
self.__failure = None
self.__stop_args = None
self.__timeout = None
# end::tornado.testing.AsyncTestCase
self._handler_exc_info = None
app.update_settings({
'TESTING': True,
'PROCESSES': 1
})
# redo setup to make settings changes effective
app.setup()
self.app = app
self.setup()
def setup(self):
"""Setup facilities for running the server
NOTE if you get all "Address already in use" error except the first one,
the bug may well happened in this function.
"""
# start::tornado.testing.AsyncTestCase
#self.io_loop = self.get_new_ioloop()
# end::tornado.testing.AsyncTestCase
#self.app.io_loop = self.io_loop
# init app.io_loop, app.application and app.http_server
self.app._init_application()
self.app.make_http_server()
self.io_loop = self.app.io_loop
if self.raise_handler_exc:
self.patch_app_handlers()
self.http_server = self.app.http_server
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
def close(self):
"""CLose http_server, io_loop by sequence, to ensure the environment
is cleaned up and invoking `setup` successfully within next test function
It is suggested to be called in `TestCase.tearDown`
"""
# start::tornado.testing.AsyncHTTPTestCase
self.http_server.stop()
if (not IOLoop.initialized() or
self.http_client.io_loop is not IOLoop.instance()):
self.http_client.close()
# end::tornado.testing.AsyncHTTPTestCase
# start::tornado.testing.AsyncTestCase
if (not IOLoop.initialized() or
self.io_loop is not IOLoop.instance()):
# Try to clean up any file descriptors left open in the ioloop.
# This avoids leaks, especially when tests are run repeatedly
# in the same process with autoreload (because curl does not
# set FD_CLOEXEC on its file descriptors)
self.io_loop.close(all_fds=True)
# end::tornado.testing.AsyncTestCase
def _parse_cookies(self, resp):
if COOKIE_HEADER_KEY in resp.headers:
c = SimpleCookie(resp.headers.get(COOKIE_HEADER_KEY))
elif COOKIE_HEADER_KEY.lower() in resp.headers:
c = SimpleCookie(resp.headers.get(COOKIE_HEADER_KEY.lower()))
else:
c = None
resp.cookies = c
def _add_cookies(self, cookies, kwgs):
# SimpleCookie is inherited from dict, so judge it first
if isinstance(cookies, SimpleCookie):
c = cookies
elif isinstance(cookies, dict):
c = SimpleCookie()
for k, v in cookies.items():
c[k] = v
else:
raise TypeError('cookies kwarg should be dict or SimpleCookie instance')
if 'headers' in kwgs:
headers = kwgs['headers']
else:
headers = kwgs['headers'] = {}
headers['Cookie'] = c.output().lstrip(COOKIE_HEADER_KEY + ': ')
#print 'headers', headers
def stop(self, _arg=None, **kwargs):
'''Stops the ioloop, causing one pending (or future) call to wait()
to return.
Keyword arguments or a single positional argument passed to stop() are
saved and will be returned by wait().
'''
assert _arg is None or not kwargs
self.__stop_args = kwargs or _arg
if self.__running:
self.io_loop.stop()
self.__running = False
self.__stopped = True
def wait(self, condition=None, timeout=5):
"""Runs the IOLoop until stop is called or timeout has passed.
In the event of a timeout, an exception will be thrown.
If condition is not None, the IOLoop will be restarted after stop()
until condition() returns true.
"""
if not self.__stopped:
if timeout:
def timeout_func():
try:
raise self.failureException(
'Async operation timed out after %s seconds' %
timeout)
except Exception:
self.__failure = sys.exc_info()
self.stop()
if self.__timeout is not None:
self.io_loop.remove_timeout(self.__timeout)
self.__timeout = self.io_loop.add_timeout(time.time() + timeout, timeout_func)
while True:
self.__running = True
with NullContext():
# Wipe out the StackContext that was established in
# self.run() so that all callbacks executed inside the
# IOLoop will re-run it.
self.io_loop.start()
if (self.__failure is not None or
condition is None or condition()):
break
assert self.__stopped
self.__stopped = False
self.__rethrow()
result = self.__stop_args
self.__stop_args = None
return result
def __rethrow(self):
if self.__failure is not None:
failure = self.__failure
self.__failure = None
raise_exc_info(failure)
def request(self, method, path,
data=None, json=False,
files=None,
cookies=None, **kwgs):
kwgs['method'] = method
## `path` should be utf-8 encoded string to complete requote process
#if isinstance(path, str):
# path = path.encode('utf8')
path = requote_uri(path)
# `body` must be passed if method is one of those three
if method in ['POST', 'PUT', 'PATCH']:
headers = kwgs.setdefault('headers', {})
body = ''
if files:
boundary = '1234567890'
headers['Content-Type'] = 'multipart/form-data; boundary=%s' % boundary
L = []
if data:
for k, v in data.items():
L.append('--' + boundary)
L.append('Content-Disposition: form-data; name="%s"' % k)
L.append('')
L.append(v)
for k, f in files.items():
L.append('--' + boundary)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (k, f[0]))
L.append('Content-Type: %s' % mimetypes.guess_type(f[0])[0] or 'application/octet-stream')
L.append('')
L.append(f[1])
L.append('--%s--' % boundary)
L.append('')
body = '\r\n'.join(L)
else:
if data:
if json:
body = json_encode(data)
headers['Content-Type'] = 'application/json'
else:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
body = urlencode(data)
kwgs['body'] = body
else:
if data:
path = '%s?%s' % (path, urlencode(data))
if cookies:
self._add_cookies(cookies, kwgs)
#print 'fetch kwgs', kwgs
url = self.get_url(path)
logging.debug('testing fetch url: %s', url)
self.http_client.fetch(url, self.stop, **kwgs)
resp = self.wait()
self._parse_cookies(resp)
return resp
def get(self, *args, **kwgs):
return self.request('GET', *args, **kwgs)
def delete(self, *args, **kwgs):
return self.request('DELETE', *args, **kwgs)
def post(self, *args, **kwgs):
return self.request('POST', *args, **kwgs)
def put(self, *args, **kwgs):
return self.request('PUT', *args, **kwgs)
def patch(self, *args, **kwgs):
return self.request('PATCH', *args, **kwgs)
def get_protocol(self):
return 'http'
def get_http_port(self):
return self.app.settings['PORT']
def get_url(self, path):
"""Returns an absolute url for the given path on the test server."""
return '%s://localhost:%s%s' % (self.get_protocol(),
self.get_http_port(), path)
def get_handler_exc(self):
if self._handler_exc_info:
raise_exc_info(self._handler_exc_info)
def patch_handler(self, hdr):
if not isinstance(hdr, web.StaticFileHandler):
hdr._testing_app_client = self
hdr._handle_request_exception = _handle_request_exception
def patch_app_handlers(self):
for host_pattern, rules in self.app.application.handlers:
for r in rules:
self.patch_handler(r.handler_class)
def post(self, path):
if 'admin-point' not in self.current_user['roles']:
self.finish()
return
user_id = self.json_args['user_id']
_type = self.json_args['type']
operator = self.json_args['operator']
token = self.json_args['token']
income = self.json_args['income']
notes = self.json_args['notes']
if _type == 'refund-manual':
order_id = self.json_args['order_id']
try:
request_log.info('CHECK REFUND ORDER %s', order_id)
order_coll = self.application.glados_client.GLaDOS.order
order = yield order_coll.find_one({'_id': order_id})
if order is None:
raise RuntimeError('无法查询订单')
if order['user_id'] != user_id:
raise RuntimeError('您查询的订单与选择的用户不一致，请核实。')
if order['value'] != int(float(income) * 10000):
# print(int(float(income) * 10000))
# print(order.value)
raise RuntimeError('订单金额异常')
if order['back_result'] not in ['1', '00000']:
raise RuntimeError('订单状态不是正常完成状态')
t = datetime.strptime(order_id[1:9], '%Y%m%d')
request_log.info('CHECK REFUND FROM %s', str(t))
trans_coll = self.application.glados_client.GLaDOS.transaction
cursor = trans_coll.find({
'order_id': order_id,
'user_id': user_id,
'type': 'refund-manual',
'create_time': {
'$gt': t
}
})
refund_trans = yield cursor.to_list(10)
if refund_trans:
return self.finish(
json.dumps({
'status': 'fail',
'msg': '该订单已经存在手工退款，请核实！'
}))
request_log.info('CHECK REFUND ORDER %s DONE', order_id)
except RuntimeError as re:
return self.finish(
json.dumps({
'status': 'fail',
'msg': str(re)
}))
except Exception as e:
request_log.exception('FUND FAIL')
return self.finish(json.dumps({'status': 'fail'}))
else:
order_id = ''
sign0 = signature(
'{user_id}{order_id}{income}{operator}{token}'.format(
user_id=user_id,
order_id=order_id,
income=income,
operator=operator,
token=token))
body = json.dumps({
'type': _type,
'order_id': order_id,
'user_id': user_id,
'income': income,
'operator': operator,
'token': token,
'notes': notes,
'sign': sign0
})
http_client = AsyncHTTPClient()
try:
downstream = self.application.config['downstream'][user_id]
url = downstream['shard']
msg = {'status': 'fail'}
request = HTTPRequest(url="http://%s/admin/fund" % url,
method='POST',
body=body,
headers={'Content-Type': 'application/json'})
response = yield http_client.fetch(request)
if response.code == 200:
request_log.info(response.body)
msg = json.loads(response.body.decode('utf8'))
if msg['status'] == 'ok':
msg = {
'status': 'ok',
'msg': '财务操作成功，用户当前余额为' + str(int(msg['msg']) / 10000)
}
except Exception as e:
request_log.exception('FUND FAIL2')
msg = {'status': 'fail', 'msg': str(e)}
finally:
http_client.close()
self.finish(json.dumps(msg))
def up_cmcc_sx(handler, partner):
handler.up_req_time = time.localtime()
appKey = partner["appKey"]
timeStamp = (datetime.now() +
timedelta(seconds=0)).strftime("%Y%m%d%H%M%S")
login_no = partner["login_no"]
acc_nbr = partner["acc_nbr"]
group_id = partner["group_id"]
member_role_id = partner["member_role_id"]
gprs_accept = group_id + timeStamp + "000001"
prod_prcid = None
k = 'private:cmcc_sx:{carrier}:{price}'.format(carrier=handler.carrier,
price=handler.price)
prod_prcid = handler.slave.get(k)
if prod_prcid is None:
handler.up_result = 5003
return handler.up_result
data = ('acc_nbr={acc_nbr}&'
'appKey={appKey}&'
'gprs_accept={gprs_accept}&'
'group_id={group_id}&'
'login_no={login_no}&'
'member_role_id={member_role_id}&'
'prod_prcid={prod_prcid}&'
'timeStamp={timeStamp}').format(appKey=appKey,
timeStamp=timeStamp,
login_no=login_no,
acc_nbr=acc_nbr,
prod_prcid=prod_prcid,
group_id=group_id,
member_role_id=member_role_id,
gprs_accept=gprs_accept)
sign = to_md5(quote(data))
rsa_sign = to_para(sign)
body = '&'.join([
"acc_nbr=%s" % quote(acc_nbr),
"appKey=%s" % quote(appKey),
"gprs_accept=%s" % quote(gprs_accept),
"group_id=%s" % quote(group_id),
"login_no=%s" % quote(login_no),
"member_role_id=%s" % quote(member_role_id),
"prod_prcid=%s" % quote(prod_prcid),
"timeStamp=%s" % quote(timeStamp),
"sign=%s" % quote(rsa_sign),
])
url = partner['url_auth']
url = url + '?' + body
h = {'Content-Type': 'application/json; charset=utf-8'}
result = 9999
up_result = None
http_client = AsyncHTTPClient()
try:
request_log.info("REQU %s", body, extra={'orderid': handler.order_id})
response = yield http_client.fetch(url,
method='GET',
headers=h,
request_timeout=120)
except HTTPError as http_error:
request_log.error('CALL UPSTREAM FAIL %s',
http_error,
extra={'orderid': handler.order_id})
result = 60000 + http_error.code
response = None
except Exception as e:
request_log.error('CALL UPSTREAM FAIL %s',
e,
extra={'orderid': handler.order_id})
response = None
finally:
http_client.close()
handler.up_resp_time = time.localtime()
if response and response.code == 200:
response_body = response.body.decode('utf8')
request_log.info("RESP %s",
response_body,
extra={'orderid': handler.order_id})
try:
response_body = json.loads(response_body)
up_result = response_body.get("resCode")
sp_order_id = response_body.get("orderId")
handler.up_order_id = sp_order_id
result = RESULT_MAP.get(up_result, 9)
if result == 1:
handler.back_result = result
yield handler.callback('1')
elif result == 9:
pass
handler.up_result = up_result
if handler.up_result == '0000000':
handler.master.set(
"map:cmcc_sx:{sp_order_id}".format(
sp_order_id=sp_order_id), handler.order_id)
except Exception as e:
result = 9999
handler.up_result = result
request_log.error('PARSE UPSTREAM %s',
e,
extra={'orderid': handler.order_id})
return result
def up_21cn(handler, partner):
"""
telecom -> up_order
"""
key = '21cn:%d' % handler.price
if 'prefix' in partner:
key = 'private:%s:%s' % (partner['prefix'], key)
if handler.scope:
key = '%s:%s' % (key, handler.scope)
prod = handler.slave.hmget(key, ['plat_offer_id', 'activity_id'])
handler.up_req_time = time.localtime()
tsp = time.strftime("%Y%m%d%H%M%S", handler.up_req_time)
passphrase = partner['pass']
iv = partner['iv']
# {"request_no":"201410211021070000000113",
# "service_code":"FS0001",
# "contract_id":"100052",
# "order_id":"0",
# "plat_offer_id":"100151", 100151:100033
# "phone_id":"18905172668",
# "channel_id":"1",
# "activity_id":"100033"}
code = json.dumps({
'request_no': handler.order_id,
'service_code': 'FS0001',
'contract_id': partner['contract_id'],
'order_id': '0',
'plat_offer_id': prod[0],
'phone_id': handler.mobile,
'channel_id': '1',
'activity_id': prod[1],
})
request_log.info('CALL_CODE %s', code, extra={'orderid': handler.order_id})
# aes
aes = AES.new(passphrase, AES.MODE_CBC, iv)
bytes = aes.encrypt(pad(code))
encrypted = hexlify(bytes)
body = json.dumps({'partner_no': partner['partner_no'], 'code': encrypted})
url = partner['url.order']
# print(handler.order_id + ":" + body)
request_log.info('CALL_REQ %s', body, extra={'orderid': handler.order_id})
response = None
result = 99999
# call & wait
http_client = AsyncHTTPClient()
try:
response = yield http_client.fetch(url, method='POST', headers=h, body=body)
except HTTPError as http_error:
request_log.error('CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id})
result = 60000 + http_error.code
except Exception as e:
request_log.error('CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id})
finally:
http_client.close()
handler.up_resp_time = time.localtime() # <--------------
if response and response.code == 200:
body = response.body.decode('utf8')
request_log.info('CALL_RESP %s', body, extra={'orderid': handler.order_id})
try:
# {"request_no":"201410211021070000000113","result_code":"00000"}
resp = json.loads(body)
result = int(resp['result_code'])
handler.up_order_id = resp['request_no']
handler.up_cost = handler.cost
handler.up_result = result
except Exception as e:
result = 99999
request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id})
if handler.up_result is None:
handler.up_result = result
return result
def up_seek(handler, partner):
slave = handler.slave
# private data
if handler.scope is None:
k = 'private:seek:{area}:{price}'.format(area=handler.area,
price=handler.price)
else:
k = 'private:seek:{area}:{price}:{scope}'.format(area=handler.area,
price=handler.price,
scope=handler.scope)
prod = slave.hmget(k, ['prod_id', 'sum'])
if prod[0] is None:
return 99999
prod_id = prod[0]
prod_sum = prod[1]
handler.up_req_time = time.localtime()
tsp = int(time.mktime(handler.up_req_time))
back_url = urllib.parse.quote_plus(partner['callback'])
sign = ''.join([
'appkey', partner['key'], 'channelOrderNo', handler.order_id,
'customer', handler.mobile, 'notifyUrl', back_url, 'prodId', prod_id,
'prodNum1prodPayType0', 'sum', prod_sum, 'timestamp',
str(tsp), partner['secret']
])
sign = signature(sign).lower()
q = '&'.join([
'appkey=' + partner['key'], 'prodId=' + prod_id,
'customer=' + handler.mobile, 'sum=' + prod_sum, 'prodNum=1',
'prodPayType=0', 'channelOrderNo=' + handler.order_id,
'notifyUrl=' + back_url, 'timestamp=' + str(tsp), 'sign=' + sign
])
url = partner['url.order']
# print(handler.order_id + ":" + body)
full_url = url + '/r/Channel/createOrder?' + q
request_log.info('CALL_REQ %s',
full_url,
extra={'orderid': handler.order_id})
response = None
result = 99999
# call & wait
http_client = AsyncHTTPClient()
try:
response = yield http_client.fetch(full_url, method='GET')
except HTTPError as http_error:
request_log.error('CALL UPSTREAM FAIL %s',
http_error,
extra={'orderid': handler.order_id})
result = 60000 + http_error.code
except Exception as e:
request_log.error('CALL UPSTREAM FAIL %s',
e,
extra={'orderid': handler.order_id})
finally:
http_client.close()
if response and response.code == 200:
body = response.body.decode('utf8')
request_log.info('CALL_RESP1 %s',
body,
extra={'orderid': handler.order_id})
try:
# {"orderNo":"1418189340100004","orderStatus":0,"createOrderTime":"20141210132900","resultCode":"1000","resultReason":""}
resp = json.loads(body)
result = int(resp['resultCode'])
if result == 1000:
handler.up_order_id = resp['orderNo']
except Exception as e:
result = 99999
request_log.error('PARSE UPSTREAM %s',
e,
extra={'orderid': handler.order_id})
if result == 1000 and handler.up_order_id:
# SUBMIT
tsp = int(time.mktime(handler.up_req_time))
sign = ''.join([
'appkey' + partner['key'], 'orderNo' + handler.up_order_id,
'timestamp' + str(tsp), partner['secret']
])
sign = signature(sign).lower()
q = '&'.join([
'orderNo=' + handler.up_order_id, 'appkey=' + partner['key'],
'timestamp=' + str(tsp), 'sign=' + sign
])
full_url = url + '/r/Channel/submitOrder?' + q
response = None
result = 99999
# call & wait
http_client = AsyncHTTPClient()
try:
response = yield http_client.fetch(full_url, method='GET')
except HTTPError as http_error:
request_log.error('CALL UPSTREAM2 FAIL %s',
http_error,
extra={'orderid': handler.order_id})
result = 60000 + http_error.code
except Exception as e:
request_log.error('CALL UPSTREAM2 FAIL %s',
e,
extra={'orderid': handler.order_id})
finally:
http_client.close()
if response and response.code == 200:
body = response.body.decode('utf8')
request_log.info('CALL_RESP2 %s',
body,
extra={'orderid': handler.order_id})
try:
# {"orderNo":"1418189340100004","orderStatus":0,"createOrderTime":"20141210132900","resultCode":"1000","resultReason":""}
resp = json.loads(body)
result = int(resp['resultCode'])
except Exception as e:
result = 99999
request_log.error('PARSE UPSTREAM2 %s',
e,
extra={'orderid': handler.order_id})
# result mapping
result = RESULT_MAPPING.get(result, result)
handler.up_result = result
handler.up_resp_time = time.localtime() # <--------------
return result
class AsyncHTTPTestCase(AsyncTestCase):
def initialize_ioloop(self):
self.io_loop = self.get_new_ioloop()
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
def setup(self):
self.stopped = False
self.running = False
self.failure = None
self.stop_args = None
if 'get_app' in dir(self.__class__):
self.initialize_ioloop()
self.app = self.get_app()
elif hasattr(self, 'get_handler_spec') and self.get_handler_spec:
spec = self.get_handler_spec()
if spec:
if isinstance(spec, tuple):
if len(spec) == 3:
(pattern, handler, kwargs) = spec
elif len(spec) == 2:
(pattern, handler) = spec
kwargs = {}
elif isinstance(spec, URLSpec):
pattern = spec.regex.pattern
handler = spec.handler_class
kwargs = spec.kwargs
# create an isolated version of the handler
self.isolated_handler = type('IsolatedHandler', (handler,), {})
self.initialize_ioloop()
self.app = Application([
(pattern, self.isolated_handler, kwargs)
], self.get_application_settings())
if self.app:
self.port = get_unused_port()
self.http_server = HTTPServer(
self.app,
io_loop=self.io_loop,
**self.get_httpserver_options()
)
self.http_server.listen(self.port, address="0.0.0.0")
def fetch(self, path, **kwargs):
"""
Simple wrapper around ``http_client``. If the given ``path`` doesn't
start with 'http' than ``path`` is passed on to the
``http_client.fetch``
"""
self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
return self.wait()
def get_httpserver_options(self):
return {}
def get_url(self, path):
if not path.startswith('http'):
return 'http://localhost:%s%s' % (self.port, path)
return path
def teardown(self):
if 'http_server' in dir(self.__class__):
self.http_server.stop()
if 'http_client' in dir(self.__class__):
self.http_client.close()
class AsyncHTTPTestCase(AsyncTestCase):
'''A test case that starts up an HTTP server.
Subclasses must override get_app(), which returns the
tornado.web.Application (or other HTTPServer callback) to be tested.
Tests will typically use the provided self.http_client to fetch
URLs from this server.
Example::
class MyHTTPTest(AsyncHTTPTestCase):
def get_app(self):
return Application([('/', MyHandler)...])
def test_homepage(self):
# The following two lines are equivalent to
# response = self.fetch('/')
# but are shown in full here to demonstrate explicit use
# of self.stop and self.wait.
self.http_client.fetch(self.get_url('/'), self.stop)
response = self.wait()
# test contents of response
'''
def setUp(self):
super(AsyncHTTPTestCase, self).setUp()
self.__port = None
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
self._app = self.get_app()
self.http_server = HTTPServer(self._app, io_loop=self.io_loop,
**self.get_httpserver_options())
self.http_server.listen(self.get_http_port(), address="127.0.0.1")
def get_app(self):
"""Should be overridden by subclasses to return a
tornado.web.Application or other HTTPServer callback.
"""
raise NotImplementedError()
def fetch(self, path, **kwargs):
"""Convenience method to synchronously fetch a url.
The given path will be appended to the local server's host and port.
Any additional kwargs will be passed directly to
AsyncHTTPClient.fetch (and so could be used to pass method="POST",
body="...", etc).
"""
self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
return self.wait()
def get_httpserver_options(self):
"""May be overridden by subclasses to return additional
keyword arguments for HTTPServer.
"""
return {}
def get_http_port(self):
"""Returns the port used by the HTTPServer.
A new port is chosen for each test.
"""
if self.__port is None:
self.__port = get_unused_port()
return self.__port
def get_url(self, path):
"""Returns an absolute url for the given path on the test server."""
return 'http://localhost:%s%s' % (self.get_http_port(), path)
def tearDown(self):
self.http_server.stop()
self.http_client.close()
super(AsyncHTTPTestCase, self).tearDown()
class CommandHelper:
def __init__(self, config):
self.server = config.get_server()
self.debug_enabled = config.getboolean('enable_repo_debug', False)
if self.debug_enabled:
logging.warn("UPDATE MANAGER: REPO DEBUG ENABLED")
shell_command = self.server.lookup_plugin('shell_command')
self.build_shell_command = shell_command.build_shell_command
AsyncHTTPClient.configure(None, defaults=dict(user_agent="Moonraker"))
self.http_client = AsyncHTTPClient()
# GitHub API Rate Limit Tracking
self.gh_rate_limit = None
self.gh_limit_remaining = None
self.gh_limit_reset_time = None
self.gh_init_evt = Event()
# Update In Progress Tracking
self.cur_update_app = self.cur_update_id = None
def get_server(self):
return self.server
def is_debug_enabled(self):
return self.debug_enabled
def set_update_info(self, app, uid):
self.cur_update_app = app
self.cur_update_id = uid
def clear_update_info(self):
self.cur_update_app = self.cur_update_id = None
def is_app_updating(self, app_name):
return self.cur_update_app == app_name
def is_update_busy(self):
return self.cur_update_app is not None
def get_rate_limit_stats(self):
return {
'github_rate_limit': self.gh_rate_limit,
'github_requests_remaining': self.gh_limit_remaining,
'github_limit_reset_time': self.gh_limit_reset_time,
}
async def init_api_rate_limit(self):
url = "https://api.github.com/rate_limit"
while 1:
try:
resp = await self.github_api_request(url, is_init=True)
core = resp['resources']['core']
self.gh_rate_limit = core['limit']
self.gh_limit_remaining = core['remaining']
self.gh_limit_reset_time = core['reset']
except Exception:
logging.exception("Error Initializing GitHub API Rate Limit")
await tornado.gen.sleep(30.)
else:
reset_time = time.ctime(self.gh_limit_reset_time)
logging.info(
"GitHub API Rate Limit Initialized\n"
f"Rate Limit: {self.gh_rate_limit}\n"
f"Rate Limit Remaining: {self.gh_limit_remaining}\n"
f"Rate Limit Reset Time: {reset_time}, "
f"Seconds Since Epoch: {self.gh_limit_reset_time}")
break
self.gh_init_evt.set()
async def run_cmd(self,
cmd,
timeout=10.,
notify=False,
retries=1,
env=None):
cb = self.notify_update_response if notify else None
scmd = self.build_shell_command(cmd, callback=cb, env=env)
while retries:
if await scmd.run(timeout=timeout):
break
retries -= 1
if not retries:
raise self.server.error("Shell Command Error")
async def run_cmd_with_response(self, cmd, timeout=10., env=None):
scmd = self.build_shell_command(cmd, None, env=env)
result = await scmd.run_with_response(timeout, retries=5)
if result is None:
raise self.server.error(f"Error Running Command: {cmd}")
return result
async def github_api_request(self, url, etag=None, is_init=False):
if not is_init:
timeout = time.time() + 30.
try:
await self.gh_init_evt.wait(timeout)
except Exception:
raise self.server.error("Timeout while waiting for GitHub "
"API Rate Limit initialization")
if self.gh_limit_remaining == 0:
curtime = time.time()
if curtime < self.gh_limit_reset_time:
raise self.server.error(
f"GitHub Rate Limit Reached\nRequest: {url}\n"
f"Limit Reset Time: {time.ctime(self.gh_limit_remaining)}")
headers = {"Accept": "application/vnd.github.v3+json"}
if etag is not None:
headers['If-None-Match'] = etag
retries = 5
while retries:
try:
timeout = time.time() + 10.
fut = self.http_client.fetch(url,
headers=headers,
connect_timeout=5.,
request_timeout=5.,
raise_error=False)
resp = await tornado.gen.with_timeout(timeout, fut)
except Exception:
retries -= 1
msg = f"Error Processing GitHub API request: {url}"
if not retries:
raise self.server.error(msg)
logging.exception(msg)
await tornado.gen.sleep(1.)
continue
etag = resp.headers.get('etag', None)
if etag is not None:
if etag[:2] == "W/":
etag = etag[2:]
logging.info("GitHub API Request Processed\n"
f"URL: {url}\n"
f"Response Code: {resp.code}\n"
f"Response Reason: {resp.reason}\n"
f"ETag: {etag}")
if resp.code == 403:
raise self.server.error(
f"Forbidden GitHub Request: {resp.reason}")
elif resp.code == 304:
logging.info(f"Github Request not Modified: {url}")
return None
if resp.code != 200:
retries -= 1
if not retries:
raise self.server.error(
f"Github Request failed: {resp.code} {resp.reason}")
logging.info(
f"Github request error, {retries} retries remaining")
await tornado.gen.sleep(1.)
continue
# Update rate limit on return success
if 'X-Ratelimit-Limit' in resp.headers and not is_init:
self.gh_rate_limit = int(resp.headers['X-Ratelimit-Limit'])
self.gh_limit_remaining = int(
resp.headers['X-Ratelimit-Remaining'])
self.gh_limit_reset_time = float(
resp.headers['X-Ratelimit-Reset'])
decoded = json.loads(resp.body)
decoded['etag'] = etag
return decoded
async def http_download_request(self, url):
retries = 5
while retries:
try:
timeout = time.time() + 130.
fut = self.http_client.fetch(
url,
headers={"Accept": "application/zip"},
connect_timeout=5.,
request_timeout=120.)
resp = await tornado.gen.with_timeout(timeout, fut)
except Exception:
retries -= 1
logging.exception("Error Processing Download")
if not retries:
raise
await tornado.gen.sleep(1.)
continue
return resp.body
def notify_update_response(self, resp, is_complete=False):
resp = resp.strip()
if isinstance(resp, bytes):
resp = resp.decode()
notification = {
'message': resp,
'application': self.cur_update_app,
'proc_id': self.cur_update_id,
'complete': is_complete
}
self.server.send_event("update_manager:update_response", notification)
def close(self):
self.http_client.close()
def up_suopai(handler, partner):
handler.up_req_time = time.localtime()
userNo = partner['userNo']
userKey = partner['userKey']
mobile = handler.mobile
orderId = handler.order_id
k = 'private:suopai:{carrier}:{price}'.format(carrier=handler.carrier,
price=handler.price)
productNo = handler.slave.get(k)
if productNo is None:
handler.up_result = 5003
return handler.up_result
body = {
"userNo": userNo,
"userKey": userKey,
"mobile": mobile,
"productNo": productNo,
"orderId": orderId,
}
url = partner['url.order']
requ_body = urlencode(body)
url = url + '?' + requ_body
result = 9999
http_client = AsyncHTTPClient()
try:
request_log.info("SUOPAI REQ %s",
requ_body,
extra={'orderid': handler.order_id})
response = yield http_client.fetch(url,
method='GET',
request_timeout=120)
except HTTPError as http_error:
request_log.error('SUOPAI CALL UPSTREAM FAIL %s',
http_error,
extra={'orderid': handler.order_id})
result = 60000 + http_error.code
response = None
except Exception as e:
request_log.error('SUOPAI CALL UPSTREAM FAIL %s',
e,
extra={'orderid': handler.order_id})
response = None
finally:
http_client.close()
# 订单超时处理
if result == 60599:
jobId = yield query_jobid(handler, partner)
if jobId is None:
result = 9
else:
response = yield query_order_by_jobid(handler, partner, jobId)
handler.up_resp_time = time.localtime()
if response and response.code == 200:
response_body = response.body.decode('utf8')
request_log.info("RESP %s",
response_body,
extra={'orderid': handler.order_id})
try:
response_body = json.loads(response_body)
up_result = response_body.get('success')
sp_order_id = response_body.get("jobId")
handler.up_order_id = sp_order_id
result = RESULT_MAP.get(up_result, 9)
handler.up_result = up_result
if handler.up_result == 'true':
handler.master.set(
"map:suopai:{sp_order_id}".format(sp_order_id=sp_order_id),
handler.order_id)
except Exception as e:
result = 9999
handler.up_result = result
request_log.error('PARSE UPSTREAM %s',
e,
extra={'orderid': handler.order_id})
return result
def post(self, *args, **kwargs):
self.set_header('Content-Type', 'application/json')
user_id = self.get_argument("user_id", None)
session_id = self.get_argument("session_id", None)
context_id = self.get_argument("context_id", None)
application_id = self.get_argument("application_id", None)
product_id = self.get_argument("product_id", None)
_type = self.get_argument("type", None)
body = json_decode(self.request.body)
if application_id is None:
self.set_status(412)
self.finish(
json_encode(
{
"status": "error",
"message": "missing param=application_id"
}
)
)
elif session_id is None:
self.set_status(412)
self.finish(
json_encode(
{
"status": "error",
"message": "missing param=session_id"
}
)
)
elif product_id is None:
self.set_status(412)
self.finish(
json_encode(
{
"status": "error",
"message": "missing param=product_id"
}
)
)
else:
url = "%s/feedback?session_id=%s&application_id=%s&product_id=%s&type=%s" % (
CONTEXT_URL,
session_id,
application_id,
product_id,
_type
)
if user_id is not None:
url += "&user_id=%s" % user_id
if context_id is not None:
url += "&context_id=%s" % context_id
# http_client = HTTPClient()
#
# response = http_client.fetch(
# HTTPRequest(
# url=url,
# body=json_encode(body),
# method="POST"
# )
# )
#
# http_client.close()
# self.set_status(response.status)
# self.finish(response.body)
http_client = AsyncHTTPClient()
http_client.fetch(
HTTPRequest(
url=url,
body=json_encode(body),
method="POST"
),
self.context_feedback_response_handler
)
http_client.close()
class AsyncHTTPTestCase(AsyncTestCase):
"""A test case that starts up an HTTP server.
Subclasses must override get_app(), which returns the
tornado.web.Application (or other HTTPServer callback) to be tested.
Tests will typically use the provided self.http_client to fetch
URLs from this server.
Example::
class MyHTTPTest(AsyncHTTPTestCase):
def get_app(self):
return Application([('/', MyHandler)...])
def test_homepage(self):
# The following two lines are equivalent to
# response = self.fetch('/')
# but are shown in full here to demonstrate explicit use
# of self.stop and self.wait.
self.http_client.fetch(self.get_url('/'), self.stop)
response = self.wait()
# test contents of response
"""
__test__ = False
@testify.setup
def build_client(self):
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
@testify.setup
def build_server(self):
self.__port = None
# We have a much simplified implementation of cookies for our test hook.
# We're going to ignore pretty much everything like host, path, secure, expires etc.
# It's basically a dumb key-value store, represented by.... a dictionary
self.cookie_jar = {}
self._app = self.get_app()
self.http_server = HTTPServer(self._app, io_loop=self.io_loop,
**self.get_httpserver_options())
self.http_server.listen(self.get_http_port(), address="127.0.0.1")
@testify.teardown
def stop_client_server(self):
self.http_client.close()
self.http_server.stop()
def get_app(self):
"""Should be overridden by subclasses to return a
tornado.web.Application or other HTTPServer callback.
"""
raise NotImplementedError()
def fetch(self, request, **kwargs):
"""Convenience method to synchronously fetch a url.
The given path will be appended to the local server's host and port.
Any additional kwargs will be passed directly to
AsyncHTTPClient.fetch (and so could be used to pass method="POST",
body="...", etc).
"""
timeout = None
if 'timeout' in kwargs:
timeout = kwargs.pop('timeout')
if hasattr(request, 'url'):
parsed = urlparse.urlparse(request.url)
request.url = self.update_urlparsed(parsed)
else:
parsed = urlparse.urlparse(request)
request = self.update_urlparsed(parsed)
if 'headers' in kwargs and not isinstance(kwargs['headers'], tornado.httputil.HTTPHeaders):
# Upgrade our headers to a fancy object so we can add multiple values.
kwargs['headers'] = tornado.httputil.HTTPHeaders(kwargs['headers'])
else:
kwargs.setdefault('headers', tornado.httputil.HTTPHeaders())
if 'body' in kwargs and not isinstance(kwargs['body'], basestring):
kwargs['body'] = json.dumps(kwargs['body'])
kwargs['headers']['Content-Type'] = 'application/json'
if 'cookies' in kwargs:
for cookie in kwargs['cookies']:
for val in cookie.values():
kwargs['headers'].add('Cookie', val.OutputString(None))
if self.cookie_jar:
cookie = Cookie.SimpleCookie(self.cookie_jar)
for val in cookie.values():
kwargs['headers'].add('Cookie', val.OutputString(None))
self.http_client.fetch(request, self.stop, **kwargs)
res = self.wait(timeout=timeout)
for cookie_val in res.headers.get_list('set-cookie'):
cookie = Cookie.SimpleCookie()
cookie.load(cookie_val)
for val in cookie.values():
if not val.value:
del self.cookie_jar[val.key]
else:
self.cookie_jar[val.key] = val.value
return TestHTTPResponse(res)
def set_cookie(self, name, value):
self.cookie_jar[name] = value
def clear_cookie(self, name):
del self.cookie_jar[name]
def get_httpserver_options(self):
"""May be overridden by subclasses to return additional
keyword arguments for HTTPServer.
"""
return {}
def get_http_port(self):
"""Returns the port used by the HTTPServer.
A new port is chosen for each test.
"""
if self.__port is None:
self.__port = get_unused_port()
return self.__port
def get_url(self, path):
"""Returns an absolute url for the given path on the test server."""
return 'http://localhost:%s%s' % (self.get_http_port(), path)
def update_urlparsed(self, parsed):
return urlparse.urlunparse(['http', 'localhost:%s' % self.get_http_port()] + list(parsed[2:]))
