def increment_deployment_counter(bundle_id, charmworld_url):
"""Increment the deployment count in Charmworld.
If the call to Charmworld fails we log the error but don't report it.
This counter is a 'best effort' attempt but it will not impede our
deployment of the bundle.
Arguments are:
- bundle_id: the ID for the bundle in Charmworld.
- charmworld_url: the URL for charmworld, including the protocol.
If None, do nothing.
Returns True if the counter is successfully incremented else False.
"""
if charmworld_url is None:
raise gen.Return(False)
if not all((isinstance(bundle_id, basestring), isinstance(charmworld_url, basestring))):
raise gen.Return(False)
path = "metric/deployments/increment"
url = u"{}api/3/bundle/{}/{}".format(charmworld_url, urllib.quote(bundle_id), path)
logging.info("Incrementing bundle deployment count using\n{}.".format(url.encode("utf-8")))
client = AsyncHTTPClient()
# We use a GET instead of a POST since there is not request body.
try:
resp = yield client.fetch(url, callback=None)
except Exception as exc:
logging.error("Attempt to increment deployment counter failed.")
logging.error("URL: {}".format(url))
logging.exception(exc)
raise gen.Return(False)
success = bool(resp.code == 200)
raise gen.Return(success)
def get(self, path):
self.set_header('Content-Type', 'image/gif')
analytics_client_id = self.get_cookie(cookie_name) or str(uuid.uuid4())
self.set_cookie(cookie_name, analytics_client_id)
self.write(gif)
self.finish()
http_client = AsyncHTTPClient()
event = gmp.Event(category='RSS', action='view-post', label=path)
for data, headers in gmp.payloads(
tracking_id=self.settings['google_analytics_rss_id'],
client_id=analytics_client_id,
requestable=event):
try:
yield http_client.fetch(
gmp.TRACKING_URI,
method='POST',
headers=headers,
body=urllib.urlencode(data))
logging.info('Tracked RSS view for "%s"' % path)
except Exception:
logging.exception('Error tracking RSS view for "%s".' % path)
class SeedSearch(Spiderman):
"""Searches YouTube using keywords/query.
"""
functions = 'search'
def search(self):
self.done = False
self.requests_made = 1
# callback function parameters
search_key = self.search_key = str(datetime.now())
related_search = self.related_search
client = self.client
cb = lambda x: related_search(x, 0)
keywords = [str(k) for k in self.input_object['seed']]
search_query = '+'.join(keywords).replace(' ', '+')
global pages
global max_results
self.http_client = AsyncHTTPClient()
for start_index in range(1, pages):
request_url = "http://gdata.youtube.com/feeds/api/videos?q={0}&orderby=relevance&alt=jsonc&v=2&max-results={1}&start-index={2}".format(
search_query,
max_results,
start_index*25)
self.http_client.fetch(request_url, callback=cb)
def es(self, url, query={}, body={}):
http_client = AsyncHTTPClient()
if not url.startswith('/'):
url = '/'+url
for arg in query:
if not isinstance(query[arg], list):
query[arg] = [query[arg]]
try:
response = yield http_client.fetch(
'http://{}{}?{}'.format(
config['api']['elasticsearch'],
url,
utils.url_encode_tornado_arguments(query) \
if query else '',
),
method='POST' if body else 'GET',
body=utils.json_dumps(body) if body else None,
)
return utils.json_loads(response.body)
except HTTPError as e:
try:
extra = utils.json_loads(e.response.body)
except:
extra = {'error': e.response.body.decode('utf-8')}
raise exceptions.Elasticsearch_exception(
e.code,
extra,
)
def test_post_participants_valid_request(self):
db = self.get_db_client().udaan
event = dict(
eventName="dummyEvent",
currentRound=0,
)
body = dict(
names="Dummy Names",
mobileNumber="9123456789"
)
request_body = json.dumps(body)
event_id = yield db.events.insert(event)
str_event_id = str(event_id)
http_client = AsyncHTTPClient()
response = yield http_client.fetch("http://localhost:8000/api/event_management/participants", method="POST",
headers=dict(Authorization=str_event_id), body=request_body)
response_body = json.loads(response.body.decode())
self.assertEqual(response.code, 200)
self.assertEqual(response_body["status"], 200)
participant = yield db.participants.find_one({"_id": ObjectId(response_body["message"])})
yield db.participants.remove({"_id": participant["_id"]})
yield db.events.remove({"_id": event_id})
self.assertEqual(response_body["message"], participant["_id"].__str__())
def test_220_jobs(self):
client = AsyncHTTPClient()
data = {
'dataset_id': 'foo',
'job_index': 0,
}
r = yield client.fetch('http://localhost:%d/jobs'%self.port,
method='POST', body=json.dumps(data),
headers={'Authorization': b'bearer '+self.token})
self.assertEqual(r.code, 201)
ret = json.loads(r.body)
job_id = ret['result']
data2 = {'status':'failed'}
r = yield client.fetch('http://localhost:%d/datasets/%s/jobs/%s/status'%(self.port,data['dataset_id'],job_id),
method='PUT', body=json.dumps(data2),
headers={'Authorization': b'bearer '+self.token})
self.assertEqual(r.code, 200)
r = yield client.fetch('http://localhost:%d/datasets/%s/jobs/%s'%(self.port,data['dataset_id'],job_id),
headers={'Authorization': b'bearer '+self.token})
self.assertEqual(r.code, 200)
ret = json.loads(r.body)
self.assertIn('status', ret)
self.assertEqual(ret['status'], 'failed')
class ExpandSearch(Spiderman):
"""Expands youtube video network.
"""
functions = 'expand'
def search(self):
self.done = False
self.requests_made = 1
self.network = {}
# callback function parameters
search_key = self.search_key = str(datetime.now())
related_search = self.related_search
client = self.client
cb = lambda x: related_search(x, 0)
video_ids = [str(k) for k in self.input_object['seed']]
global pages
global max_results
self.http_client = AsyncHTTPClient()
for video_id in video_ids:
self.http_client.fetch("http://gdata.youtube.com/feeds/api/videos/{}/related?alt=jsonc&v=2".format(video_id),
callback=cb)
def f():
print "hi"
client=AsyncHTTPClient()
print "hello"
res=yield client.fetch('http://gfsoso.com')
print "world"
print(res)
def __sweep(self):
"""
the function called by the cleanup service
"""
#print 'candid: ' + str(len(self.__remove_candidates))
self.__log.info('cleanup service - Sweep started')
for sharing_secret in self.__remove_candidates:
#cleanup all the listeners
candidate = self.__remove_candidates[sharing_secret]
#In case a lot of actions are waiting to be executed
#and are clogged in the space, don't clean it up give it
#a chance for another sweeping period
if not candidate.is_being_processed():
self.__log.info('cleanup service - cleaning candidate for %s' % sharing_secret)
candidate.cleanup()
#notify the load balancer of the cleanup
http = AsyncHTTPClient()
load_balancer = Properties.load_balancer_url
url = '/'.join([load_balancer, 'SharingFactory',sharing_secret])
http.fetch(url, method='DELETE', callback=None)
#yield gen.Task(http.fetch, url, method = 'DELETE')
#remove if from stored sharing spaces
del(self.__sharing_spaces[sharing_secret])
else:
self.__log.info('cleanup service - skipping cleaning candidate for %s is being processed' % sharing_secret)
#now nominate every one
self.__remove_candidates.clear()
for sharing_secret in self.__sharing_spaces:
self.__remove_candidates[sharing_secret] = \
self.__sharing_spaces[sharing_secret]
self.__log.info('cleanup service - Sweep finished')
self.timer = Timer(self.SWEEP_PERIOD, self.__sweep)
self.timer.start()
def run_crawler():
"""Run a crawler iteration"""
http_client = AsyncHTTPClient()
# request OVH availablility API asynchronously
response = yield http_client.fetch(URL)
response_json = json.loads(response.body.decode('utf-8'))
if not response_json or not response_json['answer']:
return
availability = response_json['answer']['availability']
for item in availability:
# look for servers of required types in OVH availability list
if SERVER_TYPES.get(item['reference']) in config['servers']:
# make a flat list of zones where servers are available
available_zones = [e['zone'] for e in item['zones']
if e['availability'] != 'unavailable']
# iterate over all tacked zones and set availability states
for zone in config['zones']:
server = SERVER_TYPES[item['reference']]
state_id = '%s_available_in_%s' % (server, zone)
# update state for each tracked zone
text = "Server %s is available in %s" % (server, zone)
message = {
'title': "Server %s available" % server,
'text': text,
'url': "http://www.kimsufi.com/fr/index.xml"
}
update_state(state_id, zone in available_zones, message)
def test_api(self):
print("Running test")
self.mock_requests = {}
# invoke service to be tested
client = AsyncHTTPClient(self.io_loop)
client.fetch(self.test_data['service_endpoint'], self.stop)
mocks_with_assertions = [x for x in self.test_data['mocks'] if 'body' in x['mock']['request']]
for mock in mocks_with_assertions:
self.wait(timeout=30)
self.assertEqual(flatten_text(self.mock_requests[mock['mock']['name']].decode("utf-8")), flatten_text(mock['mock']['request']['body']))
#TODO: Assert request headers
response = self.wait()
# print(response)
# perform assertions
for assertion in self.test_data['assertions']:
if 'http_code' in assertion:
self.assertEqual(response.code, assertion['http_code'])
if 'response' in assertion:
self.assertEqual(flatten_text(response.body.decode("utf-8")), flatten_text(assertion['response']))
if 'content-type' in assertion:
self.assertEqual(response.headers['Content-Type'], assertion['content-type'])
def _search2(self, q, limit):
data = []
http_client = AsyncHTTPClient()
requests = [HTTPRequest(url.format(quote(q)), connect_timeout=1.0, request_timeout=60.0)
for url, _ in self.search_interfaces2]
responses = yield [http_client.fetch(r) for r in requests]
i = 0
for r in responses:
j = json.loads(r.body.decode())
n = self.search_interfaces2[i][1]
i += 1
if j['numFound'] > 0:
docs = j['docs']
if n == 'F10搜索':
stocks = j['filters'][0]['stock']
doclimit = limit['f10']
else:
stocks = j['filters'][1]['stock']
if n == '研报搜索':
doclimit = limit['research']
elif n == '新闻搜索':
doclimit = limit['news']
else:
doclimit = limit['notice']
data.append({
'name': n,
'data': {
'stock': stocks[:limit['related_stock']],
'docs': docs[:doclimit]
}
})
return data
def access_token_for_id(cls, id, callback):
"""Returns the access token for an id, acquiring a new one if necessary."""
token = Cache.get(cls.auth_cache_key_template % id)
if token:
return IOLoop.instance().add_callback(lambda: callback(token))
# If we don't have an access token cached, see if we have a refresh token
token = TokenIdMapping.lookup_refresh_token(id)
if token:
post_body = urllib.urlencode({
'client_id': Config.get('oauth', 'client-id'),
'client_secret': Config.get('oauth', 'client-secret'),
'refresh_token': token,
'grant_type': 'refresh_token',
})
http_client = AsyncHTTPClient()
return http_client.fetch(
'https://accounts.google.com/o/oauth2/token',
lambda response: cls.on_refresh_complete(response, id, callback),
method='POST',
body=post_body,
request_timeout=20.0,
connect_timeout=15.0,
)
else:
logging.error("Unable to update access token for %s, no refresh token stored.", id)
return IOLoop.instance().add_callback(lambda: callback(None))
def set_motion_detection(camera_id, enabled):
from tornado.httpclient import HTTPRequest, AsyncHTTPClient
thread_id = camera_id_to_thread_id(camera_id)
if thread_id is None:
return logging.error('could not find thread id for camera with id %s' % camera_id)
if not enabled:
_motion_detected[camera_id] = False
logging.debug('%(what)s motion detection for camera with id %(id)s' % {
'what': ['disabling', 'enabling'][enabled],
'id': camera_id})
url = 'http://127.0.0.1:7999/%(id)s/detection/%(enabled)s' % {
'id': thread_id,
'enabled': ['pause', 'start'][enabled]}
def on_response(response):
if response.error:
logging.error('failed to %(what)s motion detection for camera with id %(id)s: %(msg)s' % {
'what': ['disable', 'enable'][enabled],
'id': camera_id,
'msg': utils.pretty_http_error(response)})
else:
logging.debug('successfully %(what)s motion detection for camera with id %(id)s' % {
'what': ['disabled', 'enabled'][enabled],
'id': camera_id})
request = HTTPRequest(url, connect_timeout=_MOTION_CONTROL_TIMEOUT, request_timeout=_MOTION_CONTROL_TIMEOUT)
http_client = AsyncHTTPClient()
http_client.fetch(request, on_response)
class TaskPool:
def __init__(self, maxClients):
self._ioloop = ioloop.IOLoop()
self._httpClient = AsyncHTTPClient(self._ioloop, maxClients)
self._taskNum = 0
def run(self):
self._check()
self._ioloop.start()
def spawn(self, request, callback, **kwargs):
def wapped(response):
self._taskNum -= 1
try:
callback(response)
except:
print 'spwan error:', traceback.format_exc()
pass
self._taskNum += 1
self._httpClient.fetch(request, wapped, **kwargs)
def _check(self):
def callback():
if self._taskNum == 0:
self._ioloop.stop()
return self._check()
self._ioloop.add_callback(callback)
def call(self, callback, path, **params):
""" Fetch a result and validate before calling callback """
client = AsyncHTTPClient()
params.setdefault("ywsid", self.key)
args = urllib.urlencode(params)
url = "%s%s?%s" % (BASE, path, args)
return client.fetch(url, callback=self.wrap(callback))
def get_motion_detection(camera_id, callback):
from tornado.httpclient import HTTPRequest, AsyncHTTPClient
thread_id = camera_id_to_thread_id(camera_id)
if thread_id is None:
error = 'could not find thread id for camera with id %s' % camera_id
logging.error(error)
return callback(error=error)
url = 'http://127.0.0.1:7999/%(id)s/detection/status' % {'id': thread_id}
def on_response(response):
if response.error:
return callback(error=utils.pretty_http_error(response))
enabled = bool(response.body.lower().count('active'))
logging.debug('motion detection is %(what)s for camera with id %(id)s' % {
'what': ['disabled', 'enabled'][enabled],
'id': camera_id})
callback(enabled)
request = HTTPRequest(url, connect_timeout=_MOTION_CONTROL_TIMEOUT, request_timeout=_MOTION_CONTROL_TIMEOUT)
http_client = AsyncHTTPClient()
http_client.fetch(request, callback=on_response)
def async_fetch_future(url):
http_client = AsyncHTTPClient()
my_future = Future()
fetch_future = http_client.fetch(url)
fetch_future.add_done_callback(lambda f: my_future.set_result(f.result()))
return my_future
def asynchronous_fetch(url, callback):
http_client = AsyncHTTPClient()
def handle_res(res):
callback(res.body)
http_client.fetch(url, callback=handle_res)
def make_request1(alert_type,ticker,msg_data,reg_ids,link):
json_data = {
"collapse_key" : alert_type,
"data" : {
"data": msg_data,
"ticker" : ticker,
"link" : link,
},
"registration_ids": reg_ids,
}
url = 'https://android.googleapis.com/gcm/send'
myKey = ""
data = json.dumps(json_data)
headers = {'Content-Type': 'application/json', 'Authorization': 'key=%s' % myKey}
http_client = AsyncHTTPClient()
yield http_client.fetch(url, handle_request, data, headers)
make_request("Thank you Note! ~%s"%users["name"],"Thank you for letting us know!",
"Thanks for posting %s"%(event["name"]),[androids["reg_id"]],"findergpstracking://gpstracking#/app/setup")
def search(self, query, callback):
def cb(response):
# TODO: check response for errors
data = json.load(response.buffer)
entries = data['feed']['entry']
results = []
for entry in entries:
meta = entry['media$group']
# construct video URL (with autoplay enabled)
video = 'https://www.youtube.com/embed/{0}?autoplay=1'.format(
meta['yt$videoid']['$t'])
thumbnail = filter(lambda t: t['yt$name'] == 'default',
meta['media$thumbnail'])[0]
thumbnail = thumbnail['url']
result = SearchResult(meta['media$title']['$t'], video, self.id,
thumbnail)
results.append(result)
callback(results)
# Youtube API documentation:
# https://developers.google.com/youtube/2.0/developers_guide_protocol
client = AsyncHTTPClient()
qs = urlencode({
'q': query,
'max-results': '5',
'alt': 'json',
'v': '2'
})
client.fetch('https://gdata.youtube.com/feeds/api/videos/?' + qs, cb)
def test_broadcast(s, a, b):
ss = HTTPScheduler(s)
ss.listen(0)
s.services['http'] = ss
aa = HTTPWorker(a)
aa.listen(0)
a.services['http'] = aa
a.service_ports['http'] = aa.port
s.worker_info[a.address]['services']['http'] = aa.port
bb = HTTPWorker(b)
bb.listen(0)
b.services['http'] = bb
b.service_ports['http'] = bb.port
s.worker_info[b.address]['services']['http'] = bb.port
client = AsyncHTTPClient()
a_response = yield client.fetch('http://localhost:%d/info.json' % aa.port)
b_response = yield client.fetch('http://localhost:%d/info.json' % bb.port)
s_response = yield client.fetch('http://localhost:%d/broadcast/info.json'
% ss.port)
assert (json.loads(s_response.body.decode()) ==
{a.address: json.loads(a_response.body.decode()),
b.address: json.loads(b_response.body.decode())})
ss.stop()
aa.stop()
bb.stop()
def test_with_data(e, s, a, b):
ss = HTTPScheduler(s)
ss.listen(0)
L = e.map(inc, [1, 2, 3])
L2 = yield e._scatter(['Hello', 'world!'])
yield _wait(L)
client = AsyncHTTPClient()
response = yield client.fetch('http://localhost:%d/memory-load.json' %
ss.port)
out = json.loads(response.body.decode())
assert all(isinstance(v, int) for v in out.values())
assert set(out) == {a.address, b.address}
assert sum(out.values()) == sum(map(getsizeof,
[1, 2, 3, 'Hello', 'world!']))
response = yield client.fetch('http://localhost:%s/memory-load-by-key.json'
% ss.port)
out = json.loads(response.body.decode())
assert set(out) == {a.address, b.address}
assert all(isinstance(v, dict) for v in out.values())
assert all(k in {'inc', 'data'} for d in out.values() for k in d)
assert all(isinstance(v, int) for d in out.values() for v in d.values())
assert sum(v for d in out.values() for v in d.values()) == \
sum(map(getsizeof, [1, 2, 3, 'Hello', 'world!']))
ss.stop()
def post(self):
json_str = self.get_argument("json_msg")
# print "json_str: ",json_str
value_obj = json.loads(json_str)
com_val = COMMAND_URL_DICT[value_obj["command"]]
com_url = com_val[0]
com_func = com_val[1]
url = "http://115.28.143.67:" + str(PORT) + com_url
print "---------------------------------------"
print "request url: " + url
print "request json: " + json_str
print "---------------------------------------"
if "GET" == com_func:
request = HTTPRequest(url, "GET")
http = AsyncHTTPClient()
response = yield http.fetch(request)
print "---------------------------------------"
print "response json: " + response.body
print "---------------------------------------"
self.write(response.body)
elif "POST" == com_func:
request = HTTPRequest(url, "POST", body=json_str)
http = AsyncHTTPClient()
response = yield http.fetch(request)
print "---------------------------------------"
print "response json: " + response.body
print "---------------------------------------"
self.write(response.body)
else:
pass
def post(self, *args, **kwargs):
# TODO
# save the messages to local database
# file = self.request.files['image'][0]
# file_name = file["filename"]
# image = file['body']
text = self.get_argument("text")
data = {
"from": "Mailgun Sandbox <*****@*****.**>",
"to": "<*****@*****.**>",
"subject": "Hello Udaan",
"text": text,
}
data = urlencode(data)
client = AsyncHTTPClient()
headers_object = HTTPHeaders({"X-Mailgun-Variables": dumps({"X-Mailgun-Variables": {"password": "******"}})})
request_object = HTTPRequest("https://api.mailgun.net/v3/sandbox1713f24a60034b5ab5e7fa0ca2faa9b6.mailgun.org"
"/messages",
method="POST",
headers=headers_object,
body=data,
auth_username="******",
auth_password="******"
)
print(request_object.headers.get_list("X-Mailgun-Variables"))
response = yield client.fetch(request_object)
client.close()
print(response)
if response.code == 200:
msg = "email send successfully"
self.respond(msg, response.code)
else:
msg = "Please try again"
self.respond(msg, response.code)
def w():
http_client = AsyncHTTPClient()
if req_cnt % 2 == 0:
response = yield http_client.fetch("http://localhost:8889/wait/%s/%s" % (5, req_cnt))
else:
response = yield http_client.fetch("http://localhost:8890/wait/%s/%s" % (1, req_cnt))
print ">>>> response >>>", response, response.body, handler, handler.req_cnt
def post(self):
json_str = self.get_argument("json_msg")
url = self.get_argument("url")
method = self.get_argument("method")
url = "http://115.28.143.67:" + str(PORT) + url
print "------request url: " + url + "-----method:" + method
if method == "POST" or method == "PUT":
request = HTTPRequest(url, method, body=json_str)
http = AsyncHTTPClient()
response = yield http.fetch(request)
print "------response json: " + response.body
self.write(response.body)
if method == "DELETE":
request = HTTPRequest(url, method)
http = AsyncHTTPClient()
response = yield http.fetch(request)
print "------response json: " + response.body
self.write(response.body)
def test_with_status(e, s, a, b):
ss = HTTPScheduler(s)
ss.listen(0)
client = AsyncHTTPClient()
response = yield client.fetch('http://localhost:%d/tasks.json' %
ss.port)
out = json.loads(response.body.decode())
assert out['total'] == 0
assert out['processing'] == 0
assert out['failed'] == 0
assert out['in-memory'] == 0
assert out['ready'] == 0
assert out['waiting'] == 0
L = e.map(div, range(10), range(10))
yield _wait(L)
client = AsyncHTTPClient()
response = yield client.fetch('http://localhost:%d/tasks.json' %
ss.port)
out = json.loads(response.body.decode())
assert out['failed'] == 1
assert out['in-memory'] == 9
assert out['ready'] == 0
assert out['total'] == 10
assert out['waiting'] == 0
ss.stop()
def tsend_log(self,
message,
severity,
filename=None,
url=None,
status_code=None,
headers=None,
parameters=None,
stacktrace=False):
from tornado.httpclient import AsyncHTTPClient
d = self._build_message(
message,
severity,
filename,
url,
status_code,
headers,
parameters,
stacktrace)
# want to use the better client here.
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
AsyncHTTPClient().fetch(
self._build_url(),
lambda resp: None,
method="POST",
body=json.dumps(d),
headers=headers)
def test_unexpected_error(error, msg):
class Handler(object):
def handle(self, request):
raise error
inbound = HTTPInbound()
inbound.start(Handler())
client = AsyncHTTPClient()
req = HTTPRequest(
url='http://localhost:%s' % inbound.port,
method='POST',
headers={
headers.CALLER: 'caller',
headers.SERVICE: 'service',
headers.PROCEDURE: 'procedure',
headers.ENCODING: 'json',
headers.TTL: '10000',
},
body='',
)
with pytest.raises(HTTPError) as e:
yield client.fetch(req)
e = e.value
assert e.code >= 500 and e.code < 600
assert e.response.body == msg
async def test_root_redirect(c, s, a, b):
http_client = AsyncHTTPClient()
response = await http_client.fetch("http://localhost:%d/" %
s.http_server.port)
assert response.code == 200
assert "/status" in response.effective_url
def configure_tornado():
from tornado.httpclient import AsyncHTTPClient
AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient', max_clients=16)
def get(self):
io_loop = self.request.connection.stream.io_loop
client = AsyncHTTPClient(io_loop=io_loop)
response = yield gen.Task(client.fetch, self.get_argument('url'))
response.rethrow()
self.finish(b("got response: ") + response.body)
def camelCaseify(s):
"""convert snake_case to camelCase
For the common case where some_value is set from someValue
so we don't have to specify the name twice.
"""
return re.sub(r"_([a-z])", lambda m: m.group(1).upper(), s)
# Configure JupyterHub to use the curl backend for making HTTP requests,
# rather than the pure-python implementations. The default one starts
# being too slow to make a large number of requests to the proxy API
# at the rate required.
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
c.JupyterHub.spawner_class = "kubespawner.KubeSpawner"
# Connect to a proxy running in a different pod. Note that *_SERVICE_*
# environment variables are set by Kubernetes for Services
c.ConfigurableHTTPProxy.api_url = (
f"http://proxy-api:{os.environ['PROXY_API_SERVICE_PORT']}"
)
c.ConfigurableHTTPProxy.should_start = False
# Do not shut down user pods when hub is restarted
c.JupyterHub.cleanup_servers = False
# Check that the proxy has routes appropriately setup
c.JupyterHub.last_activity_interval = 60
def __init__(self, client=None):
self.client = client or AsyncHTTPClient()
self.github_api_url = os.environ.get('GITHUB_API_URL', 'https://api.github.com/')
self.authenticate()
def get_client(self):
client = AsyncHTTPClient()
# client = HTTPClient()
return client
def up_xiamen_zyt(handler, partner):
handler.up_req_time = time.localtime()
timeStamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
requesttime = timeStamp
msisdn = handler.mobile
orderno = handler.order_id
channelNo = partner['channelNo']
key = partner["key"]
productid = None
k = 'private:telecom800:{carrier}:{price}'.format(carrier=handler.carrier, price=handler.price)
productid = handler.slave.get(k)
if productid is None:
handler.up_result = 5003
return handler.up_result
data = 'channelNo={0}&msisdn={1}&orderno={2}&productid={4}&requesttime={3}&key={5}'.format(
channelNo, msisdn, orderno, requesttime, productid, key)
data_md5 = to_md5(data)
bytesString = data_md5.encode(encoding="utf-8")
sign = base64.b64encode(bytesString).decode('gb2312')
body = {
"channelNo": channelNo,
"msisdn": msisdn,
"productid": productid,
"requesttime": requesttime,
"orderno": orderno,
"sign": sign
}
body = urlencode(body)
url = partner["url_busi"]
result = 9999
up_result = None
http_client = AsyncHTTPClient()
try:
request_log.info("REQU %s", body, extra={'orderid': handler.order_id})
response = yield http_client.fetch(url, method='POST', body=body, request_timeout=120)
except HTTPError as http_error:
request_log.error('CALL UPSTREAM FAIL %s', http_error, extra={'orderid': handler.order_id})
result = 60000 + http_error.code
response = None
except Exception as e:
request_log.error('CALL UPSTREAM FAIL %s', e, extra={'orderid': handler.order_id})
response = None
finally:
http_client.close()
handler.up_resp_time = time.localtime()
if response and response.code == 200:
response_body = response.body.decode('utf8')
request_log.info("RESP %s", response_body, extra={'orderid': handler.order_id})
try:
response_body = json.loads(response_body)
up_result = response_body["resultcode"]
handler.up_result = str(up_result)
result = RESULT_MAP.get(up_result, 9)
except Exception as e:
result = 9999
handler.up_result = result
request_log.error('PARSE UPSTREAM %s', e, extra={'orderid': handler.order_id})
return result
async def execute_http(cls, story, line, chain, command_conf):
assert isinstance(chain, deque)
assert isinstance(chain[0], Service)
hostname = await Containers.get_hostname(story, line, chain[0].name)
args = command_conf.get('arguments', {})
body = {}
query_params = {}
path_params = {}
form_fields_count = 0
request_body_fields_count = 0
for arg in args:
value = story.argument_by_name(line, arg)
location = args[arg].get('in', 'requestBody')
if location == 'query':
cls.smart_insert(story, line, command_conf, arg, value,
query_params)
elif location == 'path':
cls.smart_insert(story, line, command_conf, arg, value,
path_params)
elif location == 'requestBody':
cls.smart_insert(story, line, command_conf, arg, value, body)
request_body_fields_count += 1
elif location == 'formBody':
# Created in StoryEventHandler.
if isinstance(value, FileFormField):
body[arg] = FileFormField(arg, value.body, value.filename,
value.content_type)
else:
body[arg] = FormField(arg, value)
form_fields_count += 1
else:
raise AsyncyError(
f'Invalid location for argument "{arg}" '
f'specified: {location}',
story=story,
line=line)
if form_fields_count > 0 and request_body_fields_count > 0:
raise AsyncyError(
f'Mixed locations are not permitted. '
f'Found {request_body_fields_count} fields, '
f'of which '
f'{form_fields_count} were in the form body',
story=story,
line=line)
method = command_conf['http'].get('method', 'post')
kwargs = {'method': method.upper()}
content_type = command_conf['http'] \
.get('contentType', 'application/json')
if method.lower() == 'post':
cls._fill_http_req_body(kwargs, content_type, body)
elif len(body) > 0:
raise AsyncyError(message=f'Parameters found in the request body, '
f'but the method is {method}',
story=story,
line=line)
port = command_conf['http'].get('port', 5000)
path = HttpUtils.add_params_to_url(
command_conf['http']['path'].format(**path_params), query_params)
url = f'http://{hostname}:{port}{path}'
story.logger.debug(f'Invoking service on {url} with payload {kwargs}')
client = AsyncHTTPClient()
response = await HttpUtils.fetch_with_retry(3, story.logger, url,
client, kwargs)
story.logger.debug(f'HTTP response code is {response.code}')
if int(response.code / 100) == 2:
content_type = response.headers.get('Content-Type')
if content_type and 'application/json' in content_type:
return ujson.loads(response.body)
else:
return cls.parse_output(command_conf, response.body, story,
line, content_type)
else:
raise AsyncyError(message=f'Failed to invoke service!',
story=story,
line=line)
def begin():
global total, cnt
conn = db.connect_torndb()
source_companies = conn.query(
"select * from source_company where source=13050 order by id desc limit %s,1000",
cnt)
if len(source_companies) == 0:
logger.info("Finish.")
exit()
for source_company in source_companies:
cnt += 1
total += 1
url = "http://www.lagou.com/gongsi/searchPosition.json?companyId=%s&pageSize=1000" % source_company[
'sourceId']
request(url,
lambda r, source_company=source_company: handle_result(
r, source_company))
conn.close()
if __name__ == "__main__":
logger.info("Start...")
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
http_client = AsyncHTTPClient(max_clients=30)
begin()
tornado.ioloop.IOLoop.instance().start()
async def when(cls, s: StreamingService, story, line: dict):
service = line[LineConstants.service]
command = line[LineConstants.command]
conf = story.app.services[s.name][ServiceConstants.config]
conf_event = Dict.find(conf, f'actions.{s.command}.events.{command}')
port = Dict.find(conf_event, f'http.port', 80)
subscribe_path = Dict.find(conf_event, 'http.subscribe.path')
subscribe_method = Dict.find(conf_event, 'http.subscribe.method',
'post')
event_args = Dict.find(conf_event, 'arguments', {})
data = {}
for key in event_args:
data[key] = story.argument_by_name(line, key)
# HACK for http - send the DNS name of the app.
if s.name == 'http':
data['host'] = story.app.app_dns
# END HACK for http.
sub_url = f'http://{s.hostname}:{port}{subscribe_path}'
story.logger.debug(f'Subscription URL - {sub_url}')
engine = f'{story.app.config.ENGINE_HOST}:' \
f'{story.app.config.ENGINE_PORT}'
query_params = urllib.parse.urlencode({
'story': story.name,
'block': line['ln'],
'app': story.app.app_id
})
sub_id = str(uuid.uuid4())
sub_body = {
'endpoint': f'http://{engine}/story/event?{query_params}',
'data': data,
'event': command,
'id': sub_id
}
body = {
'sub_id': sub_id,
'sub_url': sub_url,
'sub_method': subscribe_method.upper(),
'sub_body': sub_body,
'pod_name': s.container_name,
'app_id': story.app.app_id
}
# Why request_timeout is set to 120 seconds:
# Since this is the Synapse, Synapse does multiple internal retries,
# so we must set this to a really high value.
kwargs = {
'method': subscribe_method.upper(),
'body': json.dumps(body),
'headers': {
'Content-Type': 'application/json; charset=utf-8'
},
'request_timeout': 120
}
client = AsyncHTTPClient()
story.logger.debug(f'Subscribing to {service} '
f'from {s.command} via Synapse...')
url = f'http://{story.app.config.ASYNCY_SYNAPSE_HOST}:' \
f'{story.app.config.ASYNCY_SYNAPSE_PORT}' \
f'/subscribe'
# Okay to retry a request to the Synapse a hundred times.
response = await HttpUtils.fetch_with_retry(100, story.logger, url,
client, kwargs)
if int(response.code / 100) == 2:
story.logger.debug(f'Subscribed!')
story.app.add_subscription(sub_id, s, command, body)
else:
raise AsyncyError(
message=f'Failed to subscribe to {service} from '
f'{s.command} in {s.container_name}! '
f'http err={response.error}; code={response.code}',
story=story,
line=line)
import logging
from utils import ServerError
from tornado.escape import url_escape, url_unescape
from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPError
from tornado.httputil import HTTPHeaders
from typing import (TYPE_CHECKING, Callable, Optional, Tuple, Union, Dict,
List, Any)
if TYPE_CHECKING:
from moonraker import Server
from confighelper import ConfigHelper
from io import BufferedWriter
StrOrPath = Union[str, pathlib.Path]
MAX_BODY_SIZE = 512 * 1024 * 1024
AsyncHTTPClient.configure(None,
defaults=dict(user_agent="Moonraker"),
max_body_size=MAX_BODY_SIZE)
GITHUB_PREFIX = "https://api.github.com/"
def escape_query_string(qs: str) -> str:
parts = qs.split("&")
escaped: List[str] = []
for p in parts:
item = p.split("=", 1)
key = url_escape(item[0])
if len(item) == 2:
escaped.append(f"{key}={url_escape(item[1])}")
else:
escaped.append(key)
import pytest
import requests
from tornado.httpclient import AsyncHTTPClient, HTTPClientError
from tornado.testing import AsyncHTTPTestCase, gen_test
from unittest import TestCase
from re import search
from kafkahelpers import ReconnectingClient
import app
from utils import config
from tests.fixtures.fake_mq import FakeMQ
from tests.fixtures import StopLoopException
from utils.storage import s3 as s3_storage
from mock import patch
client = AsyncHTTPClient()
def cleanup():
sh.rm(config.TOPIC_CONFIG)
class TestContentRegex(TestCase):
"""
Test the content MIME type regex described in IPP 1.
"""
def test_valid_mime_type(self):
"""
A valid MIME type is correctly recognized.
"""
mime_types = [
def cull_idle(url,
api_token,
inactive_limit,
cull_users=False,
max_age=0,
concurrency=10):
"""Shutdown idle single-user servers
If cull_users, inactive *users* will be deleted as well.
"""
auth_header = {'Authorization': 'token %s' % api_token}
req = HTTPRequest(url=url + '/users', headers=auth_header)
now = datetime.now(timezone.utc)
client = AsyncHTTPClient()
if concurrency:
semaphore = Semaphore(concurrency)
@coroutine
def fetch(req):
"""client.fetch wrapped in a semaphore to limit concurrency"""
yield semaphore.acquire()
try:
return (yield client.fetch(req))
finally:
yield semaphore.release()
else:
fetch = client.fetch
resp = yield fetch(req)
users = json.loads(resp.body.decode('utf8', 'replace'))
futures = []
@coroutine
def handle_server(user, server_name, server):
"""Handle (maybe) culling a single server
"server" is the entire server model from the API.
Returns True if server is now stopped (user removable),
False otherwise.
"""
log_name = user['name']
if server_name:
log_name = '%s/%s' % (user['name'], server_name)
if server.get('pending'):
app_log.warning("Not culling server %s with pending %s", log_name,
server['pending'])
return False
# jupyterhub < 0.9 defined 'server.url' once the server was ready
# as an *implicit* signal that the server was ready.
# 0.9 adds a dedicated, explicit 'ready' field.
# By current (0.9) definitions, servers that have no pending
# events and are not ready shouldn't be in the model,
# but let's check just to be safe.
if not server.get('ready', bool(server['url'])):
app_log.warning("Not culling not-ready not-pending server %s: %s",
log_name, server)
return False
if server.get('started'):
age = now - parse_date(server['started'])
else:
# started may be undefined on jupyterhub < 0.9
age = None
# check last activity
# last_activity can be None in 0.9
if server['last_activity']:
inactive = now - parse_date(server['last_activity'])
else:
# no activity yet, use start date
# last_activity may be None with jupyterhub 0.9,
# which introduces the 'started' field which is never None
# for running servers
inactive = age
# CUSTOM CULLING TEST CODE HERE
# Add in additional server tests here. Return False to mean "don't
# cull", True means "cull immediately", or, for example, update some
# other variables like inactive_limit.
#
# Here, server['state'] is the result of the get_state method
# on the spawner. This does *not* contain the below by
# default, you may have to modify your spawner to make this
# work. The `user` variable is the user model from the API.
#
# if server['state']['profile_name'] == 'unlimited'
# return False
# inactive_limit = server['state']['culltime']
should_cull = (inactive is not None
and inactive.total_seconds() >= inactive_limit)
if should_cull:
app_log.info("Culling server %s (inactive for %s)", log_name,
format_td(inactive))
if max_age and not should_cull:
# only check started if max_age is specified
# so that we can still be compatible with jupyterhub 0.8
# which doesn't define the 'started' field
if age is not None and age.total_seconds() >= max_age:
app_log.info(
"Culling server %s (age: %s, inactive for %s)",
log_name,
format_td(age),
format_td(inactive),
)
should_cull = True
if not should_cull:
app_log.debug(
"Not culling server %s (age: %s, inactive for %s)",
log_name,
format_td(age),
format_td(inactive),
)
return False
if server_name:
# culling a named server
delete_url = url + "/users/%s/servers/%s" % (
quote(user['name']),
quote(server['name']),
)
else:
delete_url = url + '/users/%s/server' % quote(user['name'])
req = HTTPRequest(url=delete_url, method='DELETE', headers=auth_header)
resp = yield fetch(req)
if resp.code == 202:
app_log.warning("Server %s is slow to stop", log_name)
# return False to prevent culling user with pending shutdowns
return False
return True
@coroutine
def handle_user(user):
"""Handle one user.
Create a list of their servers, and async exec them. Wait for
that to be done, and if all servers are stopped, possibly cull
the user.
"""
# shutdown servers first.
# Hub doesn't allow deleting users with running servers.
# jupyterhub 0.9 always provides a 'servers' model.
# 0.8 only does this when named servers are enabled.
if 'servers' in user:
servers = user['servers']
else:
# jupyterhub < 0.9 without named servers enabled.
# create servers dict with one entry for the default server
# from the user model.
# only if the server is running.
servers = {}
if user['server']:
servers[''] = {
'last_activity': user['last_activity'],
'pending': user['pending'],
'url': user['server'],
}
server_futures = [
handle_server(user, server_name, server)
for server_name, server in servers.items()
]
results = yield multi(server_futures)
if not cull_users:
return
# some servers are still running, cannot cull users
still_alive = len(results) - sum(results)
if still_alive:
app_log.debug(
"Not culling user %s with %i servers still alive",
user['name'],
still_alive,
)
return False
should_cull = False
if user.get('created'):
age = now - parse_date(user['created'])
else:
# created may be undefined on jupyterhub < 0.9
age = None
# check last activity
# last_activity can be None in 0.9
if user['last_activity']:
inactive = now - parse_date(user['last_activity'])
else:
# no activity yet, use start date
# last_activity may be None with jupyterhub 0.9,
# which introduces the 'created' field which is never None
inactive = age
should_cull = (inactive is not None
and inactive.total_seconds() >= inactive_limit)
if should_cull:
app_log.info("Culling user %s (inactive for %s)", user['name'],
inactive)
if max_age and not should_cull:
# only check created if max_age is specified
# so that we can still be compatible with jupyterhub 0.8
# which doesn't define the 'started' field
if age is not None and age.total_seconds() >= max_age:
app_log.info(
"Culling user %s (age: %s, inactive for %s)",
user['name'],
format_td(age),
format_td(inactive),
)
should_cull = True
if not should_cull:
app_log.debug(
"Not culling user %s (created: %s, last active: %s)",
user['name'],
format_td(age),
format_td(inactive),
)
return False
req = HTTPRequest(url=url + '/users/%s' % user['name'],
method='DELETE',
headers=auth_header)
yield fetch(req)
return True
for user in users:
futures.append((user['name'], handle_user(user)))
for (name, f) in futures:
try:
result = yield f
except Exception:
app_log.exception("Error processing %s", name)
else:
if result:
app_log.debug("Finished culling %s", name)
class HttpClient:
def __init__(self, config: ConfigHelper) -> None:
self.server = config.get_server()
self.client = AsyncHTTPClient()
self.response_cache: Dict[str, HttpResponse] = {}
self.gh_rate_limit: Optional[int] = None
self.gh_limit_remaining: Optional[int] = None
self.gh_limit_reset_time: Optional[float] = None
def register_cached_url(self,
url: str,
etag: Optional[str] = None,
last_modified: Optional[str] = None) -> None:
headers = HTTPHeaders()
if etag is not None:
headers["etag"] = etag
if last_modified is not None:
headers["last-modified"] = last_modified
if len(headers) == 0:
raise self.server.error(
"Either an Etag or Last Modified Date must be specified")
empty_resp = HttpResponse(url, 200, b"", headers, None)
self.response_cache[url] = empty_resp
def escape_url(self, url: str) -> str:
# escape the url
match = re.match(r"(https?://[^/?#]+)([^?#]+)?(\?[^#]+)?(#.+)?", url)
if match is not None:
uri, path, qs, fragment = match.groups()
if path is not None:
uri += "/".join(
[url_escape(p, plus=False) for p in path.split("/")])
if qs is not None:
uri += "?" + escape_query_string(qs[1:])
if fragment is not None:
uri += "#" + url_escape(fragment[1:], plus=False)
url = uri
return url
async def request(self,
method: str,
url: str,
body: Optional[Union[str, List[Any], Dict[str,
Any]]] = None,
headers: Optional[Dict[str, Any]] = None,
connect_timeout: float = 5.,
request_timeout: float = 10.,
attempts: int = 1,
retry_pause_time: float = .1,
enable_cache: bool = False,
send_etag: bool = True,
send_if_modified_since: bool = True) -> HttpResponse:
cache_key = url.split("?", 1)[0]
method = method.upper()
# prepare the body if required
req_headers: Dict[str, Any] = {}
if isinstance(body, (list, dict)):
body = json.dumps(body)
req_headers["Content-Type"] = "application/json"
cached: Optional[HttpResponse] = None
if enable_cache:
cached = self.response_cache.get(cache_key)
if cached is not None and send_etag:
if cached.etag is not None and send_etag:
req_headers["If-None-Match"] = cached.etag
if cached.last_modified and send_if_modified_since:
req_headers["If-Modified-Since"] = cached.last_modified
if headers is not None:
headers.update(req_headers)
elif req_headers:
headers = req_headers
timeout = 1 + connect_timeout + request_timeout
request = HTTPRequest(url,
method,
headers,
body=body,
request_timeout=request_timeout,
connect_timeout=connect_timeout)
err: Optional[BaseException] = None
for i in range(attempts):
if i:
await asyncio.sleep(retry_pause_time)
try:
fut = self.client.fetch(request, raise_error=False)
resp = await asyncio.wait_for(fut, timeout)
except asyncio.CancelledError:
raise
except Exception as e:
err = e
else:
err = resp.error
if resp.code == 304:
err = None
if cached is None:
if enable_cache:
logging.info(
"Request returned 304, however no cached "
"item was found")
result = b""
else:
logging.debug(f"Request returned from cache: {url}")
result = cached.content
elif resp.error is not None and attempts - i != 1:
continue
else:
result = resp.body
ret = HttpResponse(url, resp.code, result, resp.headers, err)
break
else:
ret = HttpResponse(url, 500, b"", HTTPHeaders(), err)
if enable_cache and ret.is_cachable():
logging.debug(f"Caching HTTP Response: {url}")
self.response_cache[cache_key] = ret
else:
self.response_cache.pop(cache_key, None)
return ret
async def get(self,
url: str,
headers: Optional[Dict[str, Any]] = None,
**kwargs) -> HttpResponse:
if "enable_cache" not in kwargs:
kwargs["enable_cache"] = True
return await self.request("GET", url, None, headers, **kwargs)
async def post(self,
url: str,
body: Union[str, List[Any], Dict[str, Any]] = "",
headers: Optional[Dict[str, Any]] = None,
**kwargs) -> HttpResponse:
return await self.request("POST", url, body, headers, **kwargs)
async def delete(self,
url: str,
headers: Optional[Dict[str, Any]] = None,
**kwargs) -> HttpResponse:
return await self.request("DELETE", url, None, headers, **kwargs)
async def github_api_request(self,
resource: str,
attempts: int = 1,
retry_pause_time: float = .1) -> HttpResponse:
url = f"{GITHUB_PREFIX}{resource.strip('/')}"
if (self.gh_limit_reset_time is not None
and self.gh_limit_remaining == 0):
curtime = time.time()
if curtime < self.gh_limit_reset_time:
reset_time = time.ctime(self.gh_limit_reset_time)
raise self.server.error(f"GitHub Rate Limit Reached\n"
f"Request: {url}\n"
f"Limit Reset Time: {reset_time}")
headers = {"Accept": "application/vnd.github.v3+json"}
resp = await self.get(url,
headers,
attempts=attempts,
retry_pause_time=retry_pause_time)
resp_hdrs = resp.headers
if 'X-Ratelimit-Limit' in resp_hdrs:
self.gh_rate_limit = int(resp_hdrs['X-Ratelimit-Limit'])
self.gh_limit_remaining = int(resp_hdrs['X-Ratelimit-Remaining'])
self.gh_limit_reset_time = float(resp_hdrs['X-Ratelimit-Reset'])
return resp
def github_api_stats(self) -> Dict[str, Any]:
return {
'github_rate_limit': self.gh_rate_limit,
'github_requests_remaining': self.gh_limit_remaining,
'github_limit_reset_time': self.gh_limit_reset_time,
}
async def get_file(
self,
url: str,
content_type: str,
connect_timeout: float = 5.,
request_timeout: float = 180.,
attempts: int = 1,
retry_pause_time: float = .1,
enable_cache: bool = False,
) -> bytes:
headers = {"Accept": content_type}
resp = await self.get(url,
headers,
connect_timeout=connect_timeout,
request_timeout=request_timeout,
attempts=attempts,
retry_pause_time=retry_pause_time,
enable_cache=enable_cache)
resp.raise_for_status()
return resp.content
async def download_file(self,
url: str,
content_type: str,
destination_path: Optional[StrOrPath] = None,
download_size: int = -1,
progress_callback: Optional[Callable[
[int, int, int], None]] = None,
connect_timeout: float = 5.,
request_timeout: float = 180.,
attempts: int = 1,
retry_pause_time: float = 1.) -> pathlib.Path:
for i in range(attempts):
dl = StreamingDownload(self.server, destination_path,
download_size, progress_callback)
try:
fut = self.client.fetch(url,
headers={"Accept": content_type},
connect_timeout=connect_timeout,
request_timeout=request_timeout,
streaming_callback=dl.on_chunk_recd,
header_callback=dl.on_headers_recd)
timeout = connect_timeout + request_timeout + 1.
resp = await asyncio.wait_for(fut, timeout)
except asyncio.CancelledError:
raise
except Exception:
if i + 1 == attempts:
raise
await asyncio.sleep(retry_pause_time)
continue
finally:
await dl.close()
if resp.code < 400:
return dl.dest_file
raise self.server.error(f"Retries exceeded for request: {url}")
def close(self):
self.client.close()
class App(object):
"""This method can be overridden by the application. By default, it simply
prints the event."""
def switch_up(self, switch_id, ports):
print "switch_up(switch_id=%s)" % switch_id
"""This method can be overridden by the application. By default, it simply
prints the event."""
def switch_down(self, switch_id):
print "switch_down(switch_id=%s)" % switch_id
"""This method can be overridden by the application. By default, it simply
prints the event."""
def port_up(self, switch_id, port_id):
print "port_up(switch_id=%s, port_id=%d)" % (switch_id, port_id)
"""This method can be overridden by the application. By default, it simply
prints the event."""
def port_down(self, switch_id, port_id):
print "port_down(switch_id=%s, port_id=%d)" % (switch_id, port_id)
"""This method can be overridden by the application. By default, it simply
prints the event and drops the packet."""
def packet_in(self, switch_id, port_id, payload):
print "packet_in(switch_id=%s, port_id=%d)" % (switch_id, port_id)
self.pkt_out(switch_id, payload, [])
def connected(self):
print "established connection to Frenetic controller"
def packet(self, payload, protocol):
from ryu.lib.packet import packet
pkt = packet.Packet(array.array('b', payload.data))
for p in pkt:
if p.protocol_name == protocol:
return p
return None
def pkt_out(self,
switch_id,
payload,
actions,
in_port=None,
policies=None):
# Renamed actions to policies in the internal API to make it clearer, but
# kept actions keyword for backward compatibility.
_policies = policies if policies != None else actions
msg = PacketOut(switch=switch_id,
payload=payload,
policies=_policies,
in_port=in_port)
pkt_out_url = "http://%s:%s/pkt_out" % (self.frenetic_http_host,
self.frenetic_http_port)
request = HTTPRequest(pkt_out_url,
method='POST',
body=json.dumps(msg.to_json()))
return self.__http_client.fetch(request)
def config(self, compiler_options):
config_url = "http://%s:%s/config" % (self.frenetic_http_host,
self.frenetic_http_port)
request = HTTPRequest(config_url,
method='POST',
body=json.dumps(compiler_options.to_json()))
return self.__http_client.fetch(request)
def run_response(self, ftr, callback):
response = ftr.result()
if (hasattr(response, 'buffer')):
data = json.loads(response.buffer.getvalue())
ps = int(data['packets'])
bs = int(data['bytes'])
callback([ps, bs])
@return_future
def query_helper(self, ftr, callback):
f = partial(self.run_response, callback=callback)
IOLoop.instance().add_future(ftr, f)
def query(self, label):
url = "http://%s:%s/query/%s" % (self.frenetic_http_host,
self.frenetic_http_port, label)
request = HTTPRequest(url, method='GET', request_timeout=0)
response_future = self.__http_client.fetch(request)
return self.query_helper(response_future)
def run_port_stats(self, ftr, callback):
response = ftr.result()
if (hasattr(response, 'buffer')):
data = json.loads(response.buffer.getvalue())
callback(data)
@return_future
def port_stats_helper(self, ftr, callback):
f = partial(self.run_port_stats, callback=callback)
IOLoop.instance().add_future(ftr, f)
# Returns a Future where the Result is a dictionary with the
# following values: port_no, rx_packets, tx_packets, rx_bytes, tx_bytes
# rx_dropped, tx_dropped, rx_errors, tx_errors, rx_frame_error, rx_over_err,
# rx_crc_err, collisions. All of these values map to an integer
def port_stats(self, switch_id, port_id):
url = "http://%s:%s/port_stats/%s/%s" % (self.frenetic_http_host,
self.frenetic_http_port,
switch_id, port_id)
request = HTTPRequest(url, method='GET', request_timeout=0)
response_future = self.__http_client.fetch(request)
return self.port_stats_helper(response_future)
@gen.coroutine
def current_switches(self):
url = "http://%s:%s/current_switches" % (self.frenetic_http_host,
self.frenetic_http_port)
req = HTTPRequest(url, method="GET", request_timeout=0)
resp = yield self.__http_client.fetch(req)
ret = dict((x["switch_id"], x["ports"]) for x in json.loads(resp.body))
raise gen.Return(ret)
def update(self, policy):
pol_json = json.dumps(policy.to_json())
url = "http://%s:%s/%s/update_json" % (
self.frenetic_http_host, self.frenetic_http_port, self.client_id)
request = HTTPRequest(url, method='POST', body=pol_json)
return self.__http_client.fetch(request)
def __init__(self):
if not hasattr(self, 'client_id'):
self.client_id = uuid.uuid4().hex
print "No client_id specified. Using %s" % self.client_id
if not hasattr(self, 'frenetic_http_host'):
self.frenetic_http_host = "localhost"
if not hasattr(self, 'frenetic_http_port'):
self.frenetic_http_port = "9000"
self.__http_client = AsyncHTTPClient()
self.__connect()
def __connect(self):
url = "http://%s:%s/version" % (self.frenetic_http_host,
self.frenetic_http_port)
req = HTTPRequest(url, method='GET', request_timeout=0)
resp_fut = self.__http_client.fetch(req)
IOLoop.instance().add_future(resp_fut, self.__handle_connect)
def __handle_connect(self, response_future):
try:
response = response_future.result()
self.__poll_event()
self.connected()
except httpclient.HTTPError as e:
if e.code == 599:
print "Frenetic not running, re-trying...."
one_second = timedelta(seconds=1)
IOLoop.instance().add_timeout(one_second, self.__connect)
else:
raise e
def start_event_loop(self):
print "Starting the tornado event loop (does not return)."
IOLoop.instance().start()
def __poll_event(self):
url = "http://%s:%s/%s/event" % (
self.frenetic_http_host, self.frenetic_http_port, self.client_id)
req = HTTPRequest(url, method='GET', request_timeout=0)
resp_fut = self.__http_client.fetch(req)
IOLoop.instance().add_future(resp_fut, self.__handle_event)
def __handle_event(self, response):
try:
event = json.loads(response.result().body)
# For some reason, port stats are leaking into the event queue, so
# just get them out
if isinstance(event, list) or 'type' not in event:
typ = "UNKNOWN"
else:
typ = event['type']
if typ == 'switch_up':
switch_id = event['switch_id']
ports = event['ports']
self.switch_up(switch_id, ports)
elif typ == 'switch_down':
switch_id = event['switch_id']
self.switch_down(switch_id)
elif typ == 'port_up':
switch_id = event['switch_id']
port_id = event['port_id']
self.port_up(switch_id, port_id)
elif typ == 'port_down':
switch_id = event['switch_id']
port_id = event['port_id']
self.port_down(switch_id, port_id)
elif typ == 'packet_in':
pk = PacketIn(event)
self.packet_in(pk.switch_id, pk.port_id, pk.payload)
self.__poll_event()
except httpclient.HTTPError as e:
if e.code == 599:
print time.strftime(
"%c") + " Frenetic crashed, re-trying in 5 seconds...."
five_seconds = timedelta(seconds=5)
# We wait for a connect instead of going through the loop again.
IOLoop.instance().add_timeout(five_seconds, self.__connect)
else:
raise e
'http://www.google.com', 'http://www.yandex.ru', 'http://www.python.org'
]
def handle_response(response):
if response.error:
print("Error:", response.error)
else:
url = response.request.url
data = response.body
print('{} : {}...({} bytes)'.format(url, data[:10], len(data)))
ioloop.IOLoop.current().stop()
loop = ioloop.IOLoop.current()
# event loop 생성
http_client = AsyncHTTPClient()
for url in urls:
loop.add_callback(http_client.fetch, url,
lambda resp: handle_response(resp))
# event loop에 콜백 추가 : add_callback(func, args, lambda)
loop.start()
# event loop 실행, 네트워크 호출을 기다리는 동안 프로그램이 다른 일을 수행할 수 있도록 즉시 리턴이 됨
# 콜백 함수의 맨 처음 라인에서 에러를 확인하고 있음을 볼 수 있는데, 이는 콜백에서 예외를 발생시킬 수 없기 때문에 꼭 필요하다
# 콜백에서 어떤 예외라도 발생한다면, 이는 이벤트 루프와 프로그램을 빠져나가므로 모든 에러들은 예외를 일으키는(raise) 대신
# 위처럼 객체로서 전달되어야 한다. 따라서 에러를 처리하는 부분이 없다면, 모든 에러들을 묻혀갈 것임을 의미한다
# 다른 문제점은, 블로킹을 없애는 방법이 유일하게 콜백 뿐이라 매우 긴 콜백 체인을 만들 수 있다는 것이다
# 복잡하며 디버깅이 어렵다는 문제가 있다
async def get_baidu_v35(cls, request):
http = AsyncHTTPClient()
print('request to baidu')
request = await http.fetch(request)
print('request to baidu done!')
return request.body
def request_get(self, url, params, callback, user_data=None, headers=None):
url = url_concat(url,params)
client = AsyncHTTPClient()
request = HTTPRequest(url,connect_timeout=10,headers=headers)
self.log_debug(url)
client.fetch(request,functools.partial(callback,user_data))
def mock_asynchttpclient(request):
"""mock AsyncHTTPClient for recording responses"""
AsyncHTTPClient.configure(MockAsyncHTTPClient)
if not os.getenv('GITHUB_ACCESS_TOKEN'):
load_mock_responses('api.github.com')
load_mock_responses('zenodo.org')
class AsyncAuthServiceProxy(object):
def __init__(self,
service_url,
service_name=None,
timeout=HTTP_TIMEOUT,
reconnect_timeout=2,
reconnect_amount=5,
max_clients=10,
max_buffer_size=104857600):
"""
:arg service_url: in format "http://{user}:{password}@{host}:{port}"
:arg service_name: TBD
:arg timeout: TBD
:arg reconnect_timeout: TBD
:arg reconnect_amount: TBD
:arg int max_clients: max_clients is the number of concurrent
requests that can be in progress. Look tornado's docs for
SimpleAsyncHTTPClient
:arg string max_buffer_size: is the number of bytes that can be
read by IOStream. It defaults to 100mb. Look tornado's docs for
SimpleAsyncHTTPClient
"""
self.__service_url = service_url
self.__reconnect_timeout = reconnect_timeout
self.__reconnect_amount = reconnect_amount or 1
self.__service_name = service_name
self.__url = urlparse.urlparse(service_url)
self.__http_client = AsyncHTTPClient(max_clients=max_clients,
max_buffer_size=max_buffer_size)
self.__id_count = 0
(user, passwd) = (self.__url.username, self.__url.password)
try:
user = user.encode('utf8')
except AttributeError:
pass
try:
passwd = passwd.encode('utf8')
except AttributeError:
pass
authpair = user + b':' + passwd
self.__auth_header = b'Basic ' + base64.b64encode(authpair)
def __getattr__(self, name):
if name.startswith('__') and name.endswith('__'):
# Python internal stuff
raise AttributeError
if self.__service_name is not None:
name = "%s.%s" % (self.__service_name, name)
return AsyncAuthServiceProxy(self.__service_url, name)
@gen.coroutine
def __call__(self, *args):
self.__id_count += 1
postdata = json.dumps({
'version': '1.1',
'method': self.__service_name,
'params': args,
'id': self.__id_count
})
headers = {
'Host': self.__url.hostname,
'User-Agent': USER_AGENT,
'Authorization': self.__auth_header,
'Content-type': 'application/json'
}
req = HTTPRequest(url=self.__service_url,
method="POST",
body=postdata,
headers=headers)
for i in range(self.__reconnect_amount):
try:
if i > 0:
l.warning("Reconnect try #{0}".format(i + 1))
response = yield self.__http_client.fetch(req)
break
except HTTPError:
err_msg = 'Failed to connect to {0}:{1}'.format(
self.__url.hostname, self.__url.port)
rtm = self.__reconnect_timeout
if rtm:
err_msg += ". Waiting {0} seconds.".format(rtm)
l.exception(err_msg)
if rtm:
io_loop = ioloop.IOLoop.current()
yield gen.Task(io_loop.add_timeout, timedelta(seconds=rtm))
else:
l.error("Reconnect tries exceed.")
return
response = json.loads(response.body, parse_float=decimal.Decimal)
if response['error'] is not None:
raise JSONRPCException(response['error'])
elif 'result' not in response:
raise JSONRPCException({
'code': -343,
'message': 'missing JSON-RPC result'
})
else:
raise gen.Return(response['result'])
def async_webrequest(url):
client = AsyncHTTPClient()
response = yield gen.Task(client.fetch, url)
data = response.body
raise gen.Return(data)
def get_baidu(cls, request, from_name=None):
http = AsyncHTTPClient()
print('request to baidu from {}'.format(from_name))
request = yield http.fetch(request)
print('request to baidu done!')
raise gen.Return(request.body)
if self.request.streaming_callback:
buffer = BytesIO()
else:
buffer = BytesIO(data) # TODO: don't require one big string?
response = HTTPResponse(original_request,
self.code, reason=getattr(self, 'reason', None),
headers=self.headers,
request_time=self.io_loop.time() - self.start_time,
start_time=self.start_wall_time,
buffer=buffer,
effective_url=self.request.url)
self._run_callback(response)
self._on_end_request()
def _on_end_request(self):
self.stream.close()
def data_received(self, chunk):
if self._should_follow_redirect():
# We're going to follow a redirect so just discard the body.
return
if self.request.streaming_callback is not None:
self.request.streaming_callback(chunk)
else:
self.chunks.append(chunk)
if __name__ == "__main__":
AsyncHTTPClient.configure(SimpleAsyncHTTPClient)
main()
class TokenManager(object):
def __init__(self,
token_endpoint,
client_id,
client_secret,
http_options=None,
**kwargs):
self._token_endpoint = token_endpoint
self._client_id = client_id
self._client_secret = client_secret
self.request_body = kwargs
self._token = Token()
self._http_options = http_options if http_options else {}
self._http_client = AsyncHTTPClient()
def _has_token(self):
return self._token.is_valid()
@gen.coroutine
def get_token(self):
if not self._has_token():
yield self._update_token()
raise gen.Return(self._token.access_token)
@gen.coroutine
def _get_token_data(self):
token_data = yield self._request_token()
raise gen.Return(token_data)
def reset_token(self):
self._token = Token()
@gen.coroutine
def _update_token(self):
token_data = yield self._get_token_data()
self._token = Token(token_data.get('access_token', ''),
token_data.get('expires_in', 0))
@gen.coroutine
def _request_token(self):
if not self._token_endpoint:
raise TokenError('Missing token endpoint')
data = self.request_body
if not data.get('grant_type'):
data['grant_type'] = 'client_credentials'
if data.get('has_body_authentication'):
data['client_id'] = self._client_id
data['client_secret'] = self._client_secret
token_data = yield self._fetch(
url=self._token_endpoint,
method="POST",
auth=(self._client_id, self._client_secret),
data=data,
)
raise gen.Return(token_data)
@gen.coroutine
def _fetch(self, url, method="GET", data=None, auth=None):
if type(data) == dict:
data = urlencode(data)
request_data = dict(url=url, headers={}, method=method, body=data)
if auth is not None:
try:
passhash = b64encode(':'.join(auth).encode('ascii'))
except TypeError as e:
raise TokenError(
'Missing credentials (client_id:client_secret)', str(e))
request_data['headers']['Authorization'] = (
'Basic %s' % passhash.decode('utf-8'))
request_data.update(self._http_options)
request = HTTPRequest(**request_data)
logger.debug('Request: %s %s', request.method, request.url)
for header in request.headers:
logger.debug('Header %s: %s', header, request.headers[header])
try:
response = yield self._http_client.fetch(request)
except HTTPError as e:
raise TokenHTTPError('Failed to request token', e.response)
result = json.loads(response.body.decode("utf-8"))
raise gen.Return(result)
def main():
"""A simple test runner.
This test runner is essentially equivalent to `unittest.main` from
the standard library, but adds support for tornado-style option
parsing and log formatting.
The easiest way to run a test is via the command line::
python -m tornado.testing tornado.test.stack_context_test
See the standard library unittest module for ways in which tests can
be specified.
Projects with many tests may wish to define a test script like
tornado/test/runtests.py. This script should define a method all()
which returns a test suite and then call tornado.testing.main().
Note that even when a test script is used, the all() test suite may
be overridden by naming a single test on the command line::
# Runs all tests
tornado/test/runtests.py
# Runs one test
tornado/test/runtests.py tornado.test.stack_context_test
"""
from tornado.options import define, options, parse_command_line
define('autoreload', type=bool, default=False,
help="DEPRECATED: use tornado.autoreload.main instead")
define('httpclient', type=str, default=None)
define('exception_on_interrupt', type=bool, default=True,
help=("If true (default), ctrl-c raises a KeyboardInterrupt "
"exception. This prints a stack trace but cannot interrupt "
"certain operations. If false, the process is more reliably "
"killed, but does not print a stack trace."))
argv = [sys.argv[0]] + parse_command_line(sys.argv)
if options.httpclient:
from tornado.httpclient import AsyncHTTPClient
AsyncHTTPClient.configure(options.httpclient)
if not options.exception_on_interrupt:
signal.signal(signal.SIGINT, signal.SIG_DFL)
if __name__ == '__main__' and len(argv) == 1:
print >> sys.stderr, "No tests specified"
sys.exit(1)
try:
# In order to be able to run tests by their fully-qualified name
# on the command line without importing all tests here,
# module must be set to None. Python 3.2's unittest.main ignores
# defaultTest if no module is given (it tries to do its own
# test discovery, which is incompatible with auto2to3), so don't
# set module if we're not asking for a specific test.
if len(argv) > 1:
unittest.main(module=None, argv=argv)
else:
unittest.main(defaultTest="all", argv=argv)
except SystemExit, e:
if e.code == 0:
logging.info('PASS')
else:
logging.error('FAIL')
if not options.autoreload:
raise
def up_esai(handler, partner):
handler.up_req_time = time.localtime()
tsp = time.strftime("%Y-%m-%d %H:%M:%S", handler.up_req_time)
# t2 = time.strftime("%Y%m%d%H%M%S", handler.up_req_time)
price = handler.price
in_order = 'IP%s%s%s' % (partner['user_number'], handler.order_id[1:15],
handler.order_id[-4:])
# sign UserNumber + PhoneNumber+ Province + City + PhoneClass + PhoneMoney(正整数) +Time + Sign
sign = signature(partner['user_number'], in_order, handler.order_id,
handler.mobile, 'Auto', 'Auto', 'Auto', price, 'None',
tsp, '600', partner['key'])
body = '&'.join([
'UserNumber=%s' % partner['user_number'],
'InOrderNumber=%s' % in_order,
'OutOrderNumber=%s' % handler.order_id,
'PhoneNumber=%s' % handler.mobile, 'Province=Auto', 'City=Auto',
'PhoneClass=Auto',
'PhoneMoney=%s' % price, 'SellPrice=None',
'StartTime=%s' % quote(tsp), 'TimeOut=600',
'RecordKey=%s' % sign[0:16], 'Remark=--'
])
url = partner['url.order']
# print(handler.order_id + ":" + body)
request_log.info('CALL_REQ %s', body, extra={'orderid': handler.order_id})
# call & wait
http_client = AsyncHTTPClient()
try:
response = yield http_client.fetch(url,
method='POST',
body=body,
connect_timeout=60,
request_timeout=60)
except Exception as e:
request_log.error('CALL UPSTREAM FAIL %s',
e,
extra={'orderid': handler.order_id})
response = None
finally:
http_client.close()
handler.up_resp_time = time.localtime() # <--------------
result = 9999
if response and response.code == 200:
body = response.body.decode('gbk')
request_log.info('CALL_RESP %s',
body,
extra={'orderid': handler.order_id})
try:
root = et.fromstring(body)
r = root.find('result').text
if r == 'success':
handler.up_order_id = root.find('inOrderNumber').text
result = 0
elif r in [
'sameorder', 'ordererr', 'attrerr', 'phoneerr', 'moneyerr',
'sellpriceerr', 'dberr'
]:
result = 5003
handler.up_result = result
except Exception as e:
result = 9999
request_log.error('PARSE UPSTREAM %s',
e,
extra={'orderid': handler.order_id})
return result
def cull_idle(url, api_token, timeout, cull_users=False):
"""Shutdown idle single-user servers
If cull_users, inactive *users* will be deleted as well.
"""
auth_header = {'Authorization': 'token %s' % api_token}
req = HTTPRequest(
url=url + '/users',
headers=auth_header,
)
now = datetime.datetime.utcnow()
cull_limit = now - datetime.timedelta(seconds=timeout)
client = AsyncHTTPClient()
resp = yield client.fetch(req)
users = json.loads(resp.body.decode('utf8', 'replace'))
futures = []
@coroutine
def cull_one(user, last_activity):
"""cull one user"""
# shutdown server first. Hub doesn't allow deleting users with running servers.
if user['server']:
app_log.info("Culling server for %s (inactive since %s)",
user['name'], last_activity)
req = HTTPRequest(
url=url + '/users/%s/server' % user['name'],
method='DELETE',
headers=auth_header,
)
resp = yield client.fetch(req)
if resp.code == 202:
msg = "Server for {} is slow to stop.".format(user['name'])
if cull_users:
app_log.warning(msg + " Not culling user yet.")
# return here so we don't continue to cull the user
# which will fail if the server is still trying to shutdown
return
app_log.warning(msg)
if cull_users:
app_log.info("Culling user %s (inactive since %s)", user['name'],
last_activity)
req = HTTPRequest(
url=url + '/users/%s' % user['name'],
method='DELETE',
headers=auth_header,
)
yield client.fetch(req)
for user in users:
if not user['server'] and not cull_users:
# server not running and not culling users, nothing to do
continue
if not user['last_activity']:
continue
last_activity = parse_date(user['last_activity'])
if last_activity < cull_limit:
# user might be in a transition (e.g. starting or stopping)
# don't try to cull if this is happening
if user['pending']:
app_log.warning("Not culling user %s with pending %s",
user['name'], user['pending'])
continue
futures.append((user['name'], cull_one(user, last_activity)))
else:
app_log.debug("Not culling %s (active since %s)", user['name'],
last_activity)
for (name, f) in futures:
yield f
app_log.debug("Finished culling %s", name)
def coroutine_fetch(url):
http_client = AsyncHTTPClient()
response = yield http_client.fetch(url)
raise gen.Return(response.body)
def fetch_corotine(url):
http_client = AsyncHTTPClient()
res = yield http_client.fetch(url)
raise gen.Return(res.body)
class AsyncHTTPTestCase(AsyncTestCase):
'''A test case that starts up an HTTP server.
Subclasses must override get_app(), which returns the
tornado.web.Application (or other HTTPServer callback) to be tested.
Tests will typically use the provided self.http_client to fetch
URLs from this server.
Example::
class MyHTTPTest(AsyncHTTPTestCase):
def get_app(self):
return Application([('/', MyHandler)...])
def test_homepage(self):
# The following two lines are equivalent to
# response = self.fetch('/')
# but are shown in full here to demonstrate explicit use
# of self.stop and self.wait.
self.http_client.fetch(self.get_url('/'), self.stop)
response = self.wait()
# test contents of response
'''
def setUp(self):
super(AsyncHTTPTestCase, self).setUp()
self.__port = None
self.http_client = AsyncHTTPClient(io_loop=self.io_loop)
self._app = self.get_app()
self.http_server = HTTPServer(self._app, io_loop=self.io_loop,
**self.get_httpserver_options())
self.http_server.listen(self.get_http_port(), address="127.0.0.1")
def get_app(self):
"""Should be overridden by subclasses to return a
tornado.web.Application or other HTTPServer callback.
"""
raise NotImplementedError()
def fetch(self, path, **kwargs):
"""Convenience method to synchronously fetch a url.
The given path will be appended to the local server's host and port.
Any additional kwargs will be passed directly to
AsyncHTTPClient.fetch (and so could be used to pass method="POST",
body="...", etc).
"""
self.http_client.fetch(self.get_url(path), self.stop, **kwargs)
return self.wait()
def get_httpserver_options(self):
"""May be overridden by subclasses to return additional
keyword arguments for HTTPServer.
"""
return {}
def get_http_port(self):
"""Returns the port used by the HTTPServer.
A new port is chosen for each test.
"""
if self.__port is None:
self.__port = get_unused_port()
return self.__port
def get_url(self, path):
"""Returns an absolute url for the given path on the test server."""
return 'http://localhost:%s%s' % (self.get_http_port(), path)
def tearDown(self):
self.http_server.stop()
self.http_client.close()
super(AsyncHTTPTestCase, self).tearDown()
