def get_page(url, charset='utf-8', decode=True):
client = HTTPClient()
try:
response = client.fetch(url)
except HTTPError as he:
#logging.debug(he)
logging.debug('HTTPError when loading "%s"', url)
return None
except Exception as e:
warnings.warn(e)
return None
if 200 <= response.code < 300:
if 'charset=' in response.headers.get('content-type'):
new_charset = response.headers.get('content-type').split(
'charset=')[1].strip()
if new_charset.lower() != charset:
logging.debug(
'charset for "%s" is reported as "%s"; override the default',
url, charset)
charset = new_charset
if decode:
return response.body.decode(charset)
else:
return response.body
else:
logging.error('cannot load address "%s" - response code is %s', url,
response.code)
return None
def get_page(url, charset='utf-8', decode=True):
client = HTTPClient()
try:
response = client.fetch(url)
except HTTPError, he:
#logging.debug(he)
logging.debug('HTTPError when loading "%s"', url)
return None
def __call__(self, path, **kw):
# convert list type arguments to something the API likes
for k, v in kw.iteritems():
if isinstance(v, _listtypes):
kw[k] = ','.join(map(str, list(v)))
cache = self._root._handler
# now send the request
path += ".xml.aspx"
if cache:
response = cache.retrieve(self._host, path, kw)
else:
response = None
if response is None:
# construct the request
request = HTTPRequest(url='https://'+self._host+path)
request.use_gzip = True
if kw:
request.body = urllib.urlencode(kw)
request.method = 'POST'
if self._proxy:
request.url = path
request.proxy_host, request.proxy_port = self._proxy
# fetch from server
http_client = HTTPClient()
response = http_client.fetch(request)
if response.code != 200:
if response.code == 404:
raise AttributeError("'%s' not available on API server (404 Not Found)" % path)
else:
raise RuntimeError("'%s' request failed (%d %s)" % (path, response.code, response.error))
# take the response's charset into account
if type(response.body) != types.UnicodeType and 'charset' in response.headers["Content-Type"]:
charset = response.headers["Content-Type"].split('charset=')[1].strip()
response = response.body.decode(charset)
else:
response = response.body
store = not not cache
else:
store = False
retrieve_fallback = cache and getattr(cache, "retrieve_fallback", False)
if retrieve_fallback:
# implementor is handling fallbacks...
try:
return _ParseXML(response, True, store and (lambda obj: cache.store(self._host, path, kw, response, obj)))
except Error, reason:
response = retrieve_fallback(self._host, path, kw, reason=e)
if response is not None:
return response
raise
def test_multi_process(self):
self.assertFalse(IOLoop.initialized())
port = get_unused_port()
def get_url(path):
return "http://127.0.0.1:%d%s" % (port, path)
sockets = bind_sockets(port, "127.0.0.1")
# ensure that none of these processes live too long
signal.alarm(5) # master process
id = fork_processes(3, max_restarts=3)
if id is None:
# back in the master process; everything worked!
self.assertTrue(task_id() is None)
for sock in sockets: sock.close()
signal.alarm(0)
return
signal.alarm(5) # child process
try:
if id in (0, 1):
signal.alarm(5)
self.assertEqual(id, task_id())
server = HTTPServer(self.get_app())
server.add_sockets(sockets)
IOLoop.instance().start()
elif id == 2:
signal.alarm(5)
self.assertEqual(id, task_id())
for sock in sockets: sock.close()
client = HTTPClient()
def fetch(url, fail_ok=False):
try:
return client.fetch(get_url(url))
except HTTPError, e:
if not (fail_ok and e.code == 599):
raise
# Make two processes exit abnormally
fetch("/?exit=2", fail_ok=True)
fetch("/?exit=3", fail_ok=True)
# They've been restarted, so a new fetch will work
int(fetch("/").body)
# Now the same with signals
# Disabled because on the mac a process dying with a signal
# can trigger an "Application exited abnormally; send error
# report to Apple?" prompt.
#fetch("/?signal=%d" % signal.SIGTERM, fail_ok=True)
#fetch("/?signal=%d" % signal.SIGABRT, fail_ok=True)
#int(fetch("/").body)
# Now kill them normally so they won't be restarted
fetch("/?exit=0", fail_ok=True)
# One process left; watch it's pid change
pid = int(fetch("/").body)
fetch("/?exit=4", fail_ok=True)
pid2 = int(fetch("/").body)
self.assertNotEqual(pid, pid2)
# Kill the last one so we shut down cleanly
fetch("/?exit=0", fail_ok=True)
os._exit(0)
except Exception:
logging.error("exception in child process %d", id, exc_info=True)
raise