def get_page(url, charset='utf-8', decode=True): client = HTTPClient() try: response = client.fetch(url) except HTTPError as he: #logging.debug(he) logging.debug('HTTPError when loading "%s"', url) return None except Exception as e: warnings.warn(e) return None if 200 <= response.code < 300: if 'charset=' in response.headers.get('content-type'): new_charset = response.headers.get('content-type').split( 'charset=')[1].strip() if new_charset.lower() != charset: logging.debug( 'charset for "%s" is reported as "%s"; override the default', url, charset) charset = new_charset if decode: return response.body.decode(charset) else: return response.body else: logging.error('cannot load address "%s" - response code is %s', url, response.code) return None
def test_multi_process(self): self.assertFalse(IOLoop.initialized()) port = get_unused_port() def get_url(path): return "http://127.0.0.1:%d%s" % (port, path) sockets = bind_sockets(port, "127.0.0.1") # ensure that none of these processes live too long signal.alarm(5) # master process id = fork_processes(3, max_restarts=3) if id is None: # back in the master process; everything worked! self.assertTrue(task_id() is None) for sock in sockets: sock.close() signal.alarm(0) return signal.alarm(5) # child process try: if id in (0, 1): signal.alarm(5) self.assertEqual(id, task_id()) server = HTTPServer(self.get_app()) server.add_sockets(sockets) IOLoop.instance().start() elif id == 2: signal.alarm(5) self.assertEqual(id, task_id()) for sock in sockets: sock.close() client = HTTPClient() def fetch(url, fail_ok=False): try: return client.fetch(get_url(url)) except HTTPError, e: if not (fail_ok and e.code == 599): raise # Make two processes exit abnormally fetch("/?exit=2", fail_ok=True) fetch("/?exit=3", fail_ok=True) # They've been restarted, so a new fetch will work int(fetch("/").body) # Now the same with signals # Disabled because on the mac a process dying with a signal # can trigger an "Application exited abnormally; send error # report to Apple?" prompt. #fetch("/?signal=%d" % signal.SIGTERM, fail_ok=True) #fetch("/?signal=%d" % signal.SIGABRT, fail_ok=True) #int(fetch("/").body) # Now kill them normally so they won't be restarted fetch("/?exit=0", fail_ok=True) # One process left; watch it's pid change pid = int(fetch("/").body) fetch("/?exit=4", fail_ok=True) pid2 = int(fetch("/").body) self.assertNotEqual(pid, pid2) # Kill the last one so we shut down cleanly fetch("/?exit=0", fail_ok=True) os._exit(0) except Exception: logging.error("exception in child process %d", id, exc_info=True) raise