def get(self):
self.name = self._check_argument('name', expect_types=(str, unicode))
urlist = ['http://www.example.com', 'http://www.google.com']
try:
for url in urlist:
request = tornado.httpclient.HTTPRequest(url,
method='GET',
connect_timeout=1,
request_timeout=5,
user_agent='TNS-SPIDER',
)
http_client = AsyncHTTPClient()
response = yield gen.Task(http_client.fetch, request)
result = self.__handle_async_request(response)
self.finish(result)
break
else:
self.finish({'e_code':ECODE.HTTP, 'e_msg': 'SUCCESS'})
pass
except HTTPError, e:
g_logger.error(e, exc_info=True)
self.api_response({'e_code':ECODE.HTTP, 'e_msg': '%s' % e})
raise StopIteration
def get(self):
self.name = self._check_argument('name', expect_types=(str, unicode))
urlist = ['http://www.example.com', 'http://www.google.com']
try:
for url in urlist:
request = tornado.httpclient.HTTPRequest(
url,
method='GET',
connect_timeout=1,
request_timeout=5,
user_agent='TNS-SPIDER',
)
http_client = AsyncHTTPClient()
response = yield gen.Task(http_client.fetch, request)
result = self.__handle_async_request(response)
self.finish(result)
break
else:
self.finish({'e_code': ECODE.HTTP, 'e_msg': 'SUCCESS'})
pass
except HTTPError, e:
g_logger.error(e, exc_info=True)
self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
raise StopIteration
def main():
''' main function
'''
# 忽略Broken Pipe信号
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
# 处理kill信号
signal.signal(signal.SIGINT, handle_signal_kill)
signal.signal(signal.SIGQUIT, handle_signal_kill)
signal.signal(signal.SIGTERM, handle_signal_kill)
signal.signal(signal.SIGHUP, handle_signal_kill)
g_logger.info('START TORNADO WEB SERVER ...')
for key, option in options.iteritems():
g_logger.info('Options: (%s, %s)', key, option.value())
try:
if sys.version_info[:3] >= (2, 5, 2):
#pycurl minimum supported version is 7.18.2
tornado.httpclient.AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient")
pass
sockets_list = []
for bind_ip in options.bind_ip.split(','):
sockets = tornado.netutil.bind_sockets(options.port,
address=bind_ip,
backlog=128)
sockets_list.append(sockets)
#tornado.process.fork_processes(0)
global g_scheduler
g_scheduler = \
tornado.ioloop.PeriodicCallback(update_global_data,
options.scheduler_interval * 1000)
g_scheduler.start()
http_server = \
tornado.httpserver.HTTPServer(xheaders=True, request_callback=TApplication())
for sockets in sockets_list:
http_server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
http_server.stop()
tornado.ioloop.IOLoop.instance().close()
g_logger.info('STOP TORNADO WEB SERVER ...')
except socket.error as e:
g_logger.warning('Socket Error: %s', e, exc_info=True)
except KeyboardInterrupt as e:
g_logger.warning('Gently Quit')
except Exception as e:
g_logger.error('UnCaught Exception: %s', e, exc_info=True)
def get(self):
try:
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name', expect_types=(str, unicode))
self.api_response({'e_code': ECODE.SUCCESS, 'e_msg': u'Hello, %s!' %
self.name})
self.async_fetch()
except HTTPError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code':ECODE.HTTP, 'e_msg': '%s' % e})
def get(self):
if __debug__:
g_logger.debug(self.request.body)
result = {'code': ECODE.SUCCESS, 'msg': EMSG.SUCCESS}
result['data'] = {}
try:
result['data'] = self.application.stat_info
except Exception, e:
g_logger.error(e, exc_info=True if __debug__ else False)
result['code'] = ECODE.DEFAULT
result['msg'] = EMSG.DEFAULT
def get(self):
if __debug__:
g_logger.debug(self.request.body)
result = {'code': ECODE.SUCCESS, 'msg': EMSG.SUCCESS}
result['data'] = {}
try:
result['data'] = self.application.stat_info
except Exception, e:
g_logger.error(e, exc_info=True if __debug__ else False)
result['code'] = ECODE.DEFAULT
result['msg'] = EMSG.DEFAULT
def get(self):
try:
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name', expect_types=(str, unicode))
self.api_response({'e_code': ECODE.SUCCESS, 'e_msg': u'Hello, %s!' %
self.name})
self.async_fetch()
except HTTPError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code':ECODE.HTTP, 'e_msg': '%s' % e})
def get(self):
try:
HttpUtil.validate_ip(self.request)
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument("name", expect_types=(str, unicode))
if self.name == "bufferx":
self.api_response({"e_code": ECODE.SUCCESS, "e_msg": u"Hello, %s!" % self.name})
else:
pass
except HTTPError, e:
g_logger.error(e)
return self.api_response({"e_code": ECODE.HTTP, "e_msg": "%s" % e})
def get(self):
try:
result = {}
result['e_code'] = ECODE.SUCCESS
result['e_msg'] = 'SUCCESS'
pop_result = self.queue.pop()
result['data'] = pop_result['data']
result['wait_time'] = pop_result['wait_time']
self.api_response(result)
except BaseError, e:
g_logger.error(e, ext_info=True)
return self.api_response({'e_code':e.e_code, 'e_msg': '%s' %
e, 'data': {}})
class HelloHandler(BaseHandler):
@tornado.web.asynchronous
def post(self):
self.get()
@tornado.web.asynchronous
def get(self):
try:
HttpUtil.validate_ip(self.request)
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name',
expect_types=(str, unicode))
if self.name == 'bufferx':
self.api_response({
'e_code': ECODE.SUCCESS,
'e_msg': u'Hello, %s!' % self.name
})
else:
pass
except HTTPError, e:
g_logger.error(e)
return self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
except BaseError, e:
g_logger.error(e)
return self.api_response({'e_code': e.e_code, 'e_msg': '%s' % e})
def get(self):
try:
HttpUtil.validate_ip(self.request)
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name',
expect_types=(str, unicode))
if self.name == 'bufferx':
self.api_response({
'e_code': ECODE.SUCCESS,
'e_msg': u'Hello, %s!' % self.name
})
else:
pass
except HTTPError, e:
g_logger.error(e)
return self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
def main():
''' main function
'''
init_options()
init_logger()
# 忽略Broken Pipe信号
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
# 处理kill信号
signal.signal(signal.SIGINT, handle_signal_kill)
signal.signal(signal.SIGQUIT, handle_signal_kill)
signal.signal(signal.SIGTERM, handle_signal_kill)
signal.signal(signal.SIGHUP, handle_signal_kill)
g_logger.info('START TORNADO SERVER ...')
for key, option in options.iteritems():
g_logger.info('Options: (%s, %s)', key, option.value())
try:
if sys.version_info[:3] >= (2, 5, 2):
#pycurl minimum supported version is 7.18.2
AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient")
pass
HTTPServer.instance().start()
tornado.ioloop.IOLoop.instance().start()
HTTPServer.instance().stop()
tornado.ioloop.IOLoop.instance().close(all_fds=True)
g_logger.info('STOP TORNADO SERVER ...')
except socket.error as e:
g_logger.warning('Socket Error: %s', e, exc_info=True)
except KeyboardInterrupt as e:
g_logger.warning('Gently Quit')
except Exception as e:
g_logger.error('UnCaught Exception: %s', e, exc_info=True)
def main():
''' main function
'''
init_options()
init_logger()
# 忽略Broken Pipe信号
signal.signal(signal.SIGPIPE, signal.SIG_IGN);
# 处理kill信号
signal.signal(signal.SIGINT, handle_signal_kill)
signal.signal(signal.SIGQUIT, handle_signal_kill)
signal.signal(signal.SIGTERM, handle_signal_kill)
signal.signal(signal.SIGHUP, handle_signal_kill)
g_logger.info('START TORNADO SERVER ...')
for key, option in options.iteritems():
g_logger.info('Options: (%s, %s)', key, option.value())
try:
if sys.version_info[:3] >= (2, 5, 2):
#pycurl minimum supported version is 7.18.2
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
pass
HTTPServer.instance().start()
tornado.ioloop.IOLoop.instance().start()
HTTPServer.instance().stop()
tornado.ioloop.IOLoop.instance().close(all_fds=True)
g_logger.info('STOP TORNADO SERVER ...')
except socket.error as e:
g_logger.warning('Socket Error: %s', e, exc_info=True)
except KeyboardInterrupt as e:
g_logger.warning('Gently Quit')
except Exception as e:
g_logger.error('UnCaught Exception: %s', e, exc_info=True)
def post(self):
try:
if not self.request.body:
return self.api_response({'e_code': ECODE.Q_ERROR, 'e_msg':
EMSG.Q_ERROR})
json_obj = tornado.escape.json_decode(self.request.body)
g_logger.info(json_obj)
if not isinstance(json_obj, dict):
return self.api_response({'e_code': ECODE.Q_TYPE_ERROR, 'e_msg':
EMSG.Q_TYPE_ERROR})
for k, v in json_obj.iteritems():
result = self.queue.push({'k': k, 'v': v})
if ECODE.Q_FULL == result:
break
pass
self.api_response({'e_code':ECODE.SUCCESS, 'e_msg': 'SUCCESS'})
except HTTPError, e:
g_logger.error(e)
return self.api_response({'e_code':ECODE.HTTP, 'e_msg': '%s' % e})
class AsyncReadHandler(BaseHandler):
@property
def db(self):
if __debug__:
g_logger.debug('call property::db')
return self.application.db
@tornado.web.asynchronous
def post(self):
self.get()
@util_decorator.validate_ip(_logger=g_logger)
@tornado.web.asynchronous
@gen.engine
def get(self):
self.name = self._check_argument('name', expect_types=(str, unicode))
urlist = ['http://www.example.com', 'http://www.google.com']
try:
for url in urlist:
request = tornado.httpclient.HTTPRequest(
url,
method='GET',
connect_timeout=1,
request_timeout=5,
user_agent='TWORK-SPIDER',
)
http_client = AsyncHTTPClient()
response = yield gen.Task(http_client.fetch, request)
result = self.__handle_async_request(response)
self.finish(result)
break
else:
self.finish({'e_code': ECODE.HTTP, 'e_msg': 'SUCCESS'})
pass
except HTTPError, e:
g_logger.error(e, exc_info=True)
self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
raise StopIteration
except BaseError, e:
g_logger.error(e, exc_info=True)
self.api_response({'e_code': e.e_code, 'e_msg': '%s' % e})
raise StopIteration
class HelloHandler(BaseHandler):
@util_decorator.time_it(g_logger)
@tornado.web.asynchronous
def get(self):
try:
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name', expect_types=(str, unicode))
self.api_response({'e_code': ECODE.SUCCESS, 'e_msg': u'Hello, %s!' %
self.name})
self.async_fetch()
except HTTPError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code':ECODE.HTTP, 'e_msg': '%s' % e})
except BaseError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code':e.e_code, 'e_msg': '%s' % e})
class QueuePushHandler(BaseHandler):
@property
def queue(self):
return self.application.queue
@tornado.web.asynchronous
def get(self):
self.post()
@tornado.web.asynchronous
@util_decorator.validate_ip(g_logger)
def post(self):
try:
if not self.request.body:
return self.api_response({'e_code': ECODE.Q_ERROR, 'e_msg':
EMSG.Q_ERROR})
json_obj = tornado.escape.json_decode(self.request.body)
g_logger.info(json_obj)
if not isinstance(json_obj, dict):
return self.api_response({'e_code': ECODE.Q_TYPE_ERROR, 'e_msg':
EMSG.Q_TYPE_ERROR})
for k, v in json_obj.iteritems():
result = self.queue.push({'k': k, 'v': v})
if ECODE.Q_FULL == result:
break
pass
self.api_response({'e_code':ECODE.SUCCESS, 'e_msg': 'SUCCESS'})
except HTTPError, e:
g_logger.error(e)
return self.api_response({'e_code':ECODE.HTTP, 'e_msg': '%s' % e})
except ValueError, e:
g_logger.error(e)
return self.api_response({'e_code': ECODE.Q_TYPE_ERROR, 'e_msg':
EMSG.Q_TYPE_ERROR})
class HelloHandler(BaseHandler):
@property
def db(self):
if __debug__:
g_logger.debug('call property::db')
return self.application.db
@tornado.web.asynchronous
def post(self):
self.get()
@util_decorator.time_it(g_logger)
@tornado.web.asynchronous
def get(self):
try:
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name',
expect_types=(str, unicode))
self.api_response({
'e_code': ECODE.SUCCESS,
'e_msg': u'Hello, %s!' % self.name
})
if __debug__:
g_logger.debug(self.db)
g_logger.debug(self._db)
self.async_fetch()
except HTTPError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
except BaseError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code': e.e_code, 'e_msg': '%s' % e})
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name', expect_types=(str, unicode))
self.api_response({'e_code': ECODE.SUCCESS, 'e_msg': u'Hello, %s!' %
self.name})
self.async_fetch()
except HTTPError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code':ECODE.HTTP, 'e_msg': '%s' % e})
except BaseError, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code':e.e_code, 'e_msg': '%s' % e})
except Exception, e:
g_logger.error(e, exc_info=True)
return self.api_response({'e_code':ECODE.DEFAULT, 'e_msg':
'Unknown'})
def async_fetch(self):
request = tornado.httpclient.HTTPRequest(url='http://www.facebook.com',
method='GET',
connect_timeout=1,
request_timeout=5,
user_agent='TNS-SPIDER',
)
http_client = AsyncHTTPClient()
http_client.fetch(request, self.__handle_async_request)
def __handle_async_request(self, response):
g_logger.debug('CURL_ERRNO\t%d', response.error.errno)
def get(self):
try:
return self.api_response(self.queue.status())
except BaseError, e:
g_logger.error(e)
return self.api_response({'e_code':e.e_code, 'e_msg': '%s' % e})
from domain.object.error import BaseError
class HelloHandler(BaseHandler):
@tornado.web.asynchronous
def post(self):
self.get()
@tornado.web.asynchronous
def get(self):
try:
HttpUtil.validate_ip(self.request)
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument("name", expect_types=(str, unicode))
if self.name == "bufferx":
self.api_response({"e_code": ECODE.SUCCESS, "e_msg": u"Hello, %s!" % self.name})
else:
pass
except HTTPError, e:
g_logger.error(e)
return self.api_response({"e_code": ECODE.HTTP, "e_msg": "%s" % e})
except BaseError, e:
g_logger.error(e)
return self.api_response({"e_code": e.e_code, "e_msg": "%s" % e})
except Exception, e:
g_logger.error(e)
return self.api_response({"e_code": ECODE.DEFAULT, "e_msg": "Unknown"})
def get(self):
try:
return self.api_response(self.queue.status())
except BaseError, e:
g_logger.error(e)
return self.api_response({'e_code': e.e_code, 'e_msg': '%s' % e})
# 处理kill信号
signal.signal(signal.SIGINT, handle_signal_kill)
signal.signal(signal.SIGQUIT, handle_signal_kill)
signal.signal(signal.SIGTERM, handle_signal_kill)
signal.signal(signal.SIGHUP, handle_signal_kill)
g_logger.info('START TORNADO WEB SERVER ...')
for key, option in options.iteritems():
g_logger.info('Options: (%s, %s)', key, option.value())
try:
http_server = \
tornado.httpserver.HTTPServer(request_callback=TApplication())
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
g_logger.info('STOP TORNADO WEB SERVER ...')
except socket.error, e:
g_logger.warning('Socket Error: %s' % str(e))
except KeyboardInterrupt, e:
g_logger.warning('Gently Quit')
except Exception, msg:
g_logger.error('UnCaught Exception: %s' % msg)
raise
if __name__ == '__main__':
main()
http_client = AsyncHTTPClient()
response = yield gen.Task(http_client.fetch, request)
result = self.__handle_async_request(response)
self.finish(result)
break
else:
self.finish({'e_code': ECODE.HTTP, 'e_msg': 'SUCCESS'})
pass
except HTTPError, e:
g_logger.error(e, exc_info=True)
self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
raise StopIteration
except BaseError, e:
g_logger.error(e, exc_info=True)
self.api_response({'e_code': e.e_code, 'e_msg': '%s' % e})
raise StopIteration
except Exception, e:
g_logger.error(e, exc_info=True)
self.api_response({'e_code': e.e_code, 'e_msg': '%s' % e})
raise StopIteration
def __handle_async_request(self, response):
if response is None:
return None
#g_logger.debug('STACK_CONTEXT\tself.name=%s' % self.name)
g_logger.debug('RESPONSE_ERROR\t%s' % response.error)
g_logger.debug('RESPONSE\t%s' % response)
return response.body
# 处理kill信号
signal.signal(signal.SIGINT, handle_signal_kill)
signal.signal(signal.SIGQUIT, handle_signal_kill)
signal.signal(signal.SIGTERM, handle_signal_kill)
signal.signal(signal.SIGHUP, handle_signal_kill)
g_logger.info('START TORNADO WEB SERVER ...')
for key, option in options.iteritems():
g_logger.info('Options: (%s, %s)', key, option.value())
try:
http_server = \
tornado.httpserver.HTTPServer(request_callback=TApplication())
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
g_logger.info('STOP TORNADO WEB SERVER ...')
except socket.error, e:
g_logger.warning('Socket Error: %s' % str(e))
except KeyboardInterrupt, e:
g_logger.warning('Gently Quit')
except Exception, msg:
g_logger.error('UnCaught Exception: %s' % msg)
raise
if __name__ == '__main__':
main()
@tornado.web.asynchronous
def get(self):
try:
HttpUtil.validate_ip(self.request)
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name',
expect_types=(str, unicode))
if self.name == 'bufferx':
self.api_response({
'e_code': ECODE.SUCCESS,
'e_msg': u'Hello, %s!' % self.name
})
else:
pass
except HTTPError, e:
g_logger.error(e)
return self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
except BaseError, e:
g_logger.error(e)
return self.api_response({'e_code': e.e_code, 'e_msg': '%s' % e})
except Exception, e:
g_logger.error(e)
return self.api_response({
'e_code': ECODE.DEFAULT,
'e_msg': 'Unknown'
})
