def start(self):
print( self._conf.get('prefix'))
self.stop()
self._mcnt = 0
self._socketId = '{0}-{1}'.format(self._conf['prefix'], uuid4())
print(self._socketId)
self._sock = self._ctx.socket(zmq.DEALER)
self._monitor = self._sock.get_monitor_socket()
self._sock.setsockopt_string(zmq.IDENTITY, self._socketId)
self._sock.setsockopt(zmq.LINGER,1)
self._stream = zmqstream.ZMQStream(self._sock, self._loop)
self._stream.on_recv_stream(self.on_data_recv)
try:
self._stream.connect(self._broker)
self._liveness = HEARTBEAT_LIVENESS
except Exception as ex:
self.on_error(['error', ex])
gen_log.exception(ex)
self._loop.spawn_callback(self.on_socket_event,self._monitor)
# self.hbeater = ioloop.PeriodicCallback(self.beat, period, self._loop)
# self.hbeater.start()
self.on_start()
def wrapper(self, *args, **kwargs):
sign = self.get_argument('sign')
user_id = self.get_argument('user_id')
remote_ip = self.request.remote_ip
request_msg = 'remote_ip: %s, params:%s, body:%s' % \
(remote_ip, self.request.query_arguments, self.request.body)
response = None
checker = Checker(redis_cli, remote_ip, user_id, sign)
try:
checker.check()
except Exception as e:
checker.fail()
response = RESP_AUTH_CHECK_ERROR
gen_log.error('request_msg[%s], err[%s]', request_msg, e)
else:
checker.success()
if not response:
try:
response = func(self, *args, **kwargs)
except Exception as e:
response = RESP_TOP_MONITOR_ERROR
gen_log.exception('request_msg[%s] err[%s]', request_msg, e)
self.write(response)
log_msg = 'request_msg[%s] ret[%s]' % (request_msg, response)
gen_log.info(log_msg)
return response
def crawl(self):
"""
:return:
"""
log.info('Sending API requests...')
while True:
try:
yield gen.sleep(self.delay)
gen_log.info('Items in queue: %d' % len(self.to_process))
if len(self.to_process) < 1:
gen_log.info(
'Campaign requests completed. API Errors: %s' %
self.api_errors)
break
# Read News providers from Queue.
source = self.to_process.pop()
gen_log.info('Provider: %s Source: %s' %
(settings.NEWS_API, source))
if self.campaign.query:
# Build News API URL for search query.
url_params = {
'q': source,
'language': DEFAULT_LANGUAGE,
'sortBy': settings.NEWS_API_SORT_ORDER,
'pageSize': settings.NEWS_PAGE_SIZE
}
news_url = '%s%s?%s' % (settings.NEWS_API_URL,
'everything',
urlencode(url_params))
else:
url_params = {
'sources': source,
'sortBy': settings.NEWS_API_SORT_ORDER,
'pageSize': settings.NEWS_PAGE_SIZE
}
if source == 'recode':
del url_params['sortBy']
news_url = '%s%s?%s' % (settings.NEWS_API_URL,
'top-headlines',
urlencode(url_params))
request = httpclient.HTTPRequest(
url=news_url,
headers=_get_auth_headers(settings.NEWS_API_KEY),
connect_timeout=CONNECTION_TIMEOUT,
request_timeout=REQUEST_TIMEOUT)
self.backlog.fetch(request,
callback=partial(self.handle_request,
source))
except httpclient.HTTPError as exception:
gen_log.exception('crawl() %r' % exception)
self.api_errors += 1
if self.handle_errors(self.api_errors):
break
except IndexError:
break
gen_log.info('Waiting for active Provider requests to complete...')
yield gen.sleep(settings.CAMPAIGN_LIMIT)
def shutdown(self, io_loop):
for func in self.funcs[:]:
try:
func()
except:
LOG.exception('Failed to call %r', func)
LOG.warn(
'Shutdown initiated, will force stop in %d seconds',
self.max_wait
)
self.stop_loop(io_loop, self.time_func() + self.max_wait)
def _callback(fut):
exc_obj = fut.exception()
if exc_obj:
if isinstance(exc_obj, HTTPError) and exc_obj.code < 500:
try:
self._handle_server_response(exc_obj.response)
except Exception, ex:
new_future.set_exception(ex)
return
elif (isinstance(exc_obj, HTTPError)
and exc_obj.code > 500) or isinstance(
exc_obj, socket.error):
_log.error("Request to server %s failed: %r",
self._base_url, exc_obj)
if isinstance(params, dict) and params.get("wait") == "true" \
and hasattr(exc_obj, 'code') and exc_obj.code == 599:
_log.debug("Watch timed out.")
exc = etcdexcept.EtcdWatchTimedOut(
"Watch timed out: %r" % exc_obj, cause=exc_obj)
new_future.set_exception(exc)
return
elif self._allow_reconnect and fut._retry:
_log.info(
"Reconnection allowed, looking for another "
"server.")
self._base_url = fut._retry.pop(0)
fut_reconnect = func(self,
path,
method,
params=params,
timeout=timeout)
fut_reconnect._retry = fut._retry
self.ioloop.add_future(fut_reconnect, _callback)
return
else:
_log.debug("Reconnection disabled, giving up.")
exc = etcdexcept.EtcdConnectionFailed(
"Connection to etcd failed due to %r" %
exc_obj,
cause=exc_obj)
new_future.set_exception(exc)
return
else:
_log.exception("Unexpected request failure.")
exc = etcdexcept.EtcdException(exc_obj.message)
new_future.set_exception(exc)
return
async def get(self):
urls = self.get_arguments('url')
http_client = AsyncHTTPClient()
for url in urls:
try:
gen_log.info(f'>>>>>准备请求url: {url}')
response = await http_client.fetch(url)
gen_log.info(f'>>>>>已获取请求结果')
except Exception as e:
msg = "Error: %s" % e
gen_log.exception(msg)
self.write(msg)
else:
gen_log.info(response.body)
self.write(response.body)
def resolve_urls_from_domain(url):
# 通过url获取域名下面的所有的ip
try:
parsed_uri = urlparse(url)
domain = parsed_uri.hostname
port = parsed_uri.port
ips = get_ips_by_domain(domain)
urls = []
for ip in ips:
urls.append("http://%s:%s%s" % (ip, port, parsed_uri.path))
return urls
except Exception as e:
gen_log.exception(e)
return [url]
def _callback(fut):
exc = fut.exc_info()
if exc:
exc_obj = exc[1]
if isinstance(exc_obj, HTTPError) and exc_obj.code < 500:
result = self._handle_server_response(exc_obj.response)
new_future.set_result(result)
return
elif (isinstance(exc_obj, HTTPError)
and exc_obj.code > 500) or isinstance(
exc_obj, socket.error):
_log.error("Request to server %s failed: %r",
self._base_url, exc_obj)
if isinstance(params,
dict) and params.get("wait") == "true":
_log.debug("Watch timed out.")
raise etcdexcept.EtcdWatchTimedOut(
"Watch timed out: %r" % exc_obj, cause=exc_obj)
if self._allow_reconnect and fut._retry:
_log.info(
"Reconnection allowed, looking for another "
"server.")
self._base_url = fut._retry.pop(0)
fut_reconnect = func(self,
path,
method,
params=params,
timeout=timeout)
fut_reconnect._retry = fut._retry
self.ioloop.add_future(fut_reconnect, _callback)
return
else:
_log.debug("Reconnection disabled, giving up.")
raise etcdexcept.EtcdConnectionFailed(
"Connection to etcd failed due to %r" %
exc_obj,
cause=exc_obj)
else:
_log.exception("Unexpected request failure.")
result = fut.result()
self._check_cluster_id(result)
result = self._handle_server_response(fut.result())
new_future.set_result(result)
def _callback(fut):
exc = fut.exc_info()
if exc:
exc_obj = exc[1]
if isinstance(exc_obj, HTTPError) and exc_obj.code < 500:
result = self._handle_server_response(exc_obj.response)
new_future.set_result(result)
return
elif (isinstance(exc_obj, HTTPError) and exc_obj.code > 500) or isinstance(exc_obj, socket.error):
_log.error("Request to server %s failed: %r",
self._base_url, exc_obj)
if isinstance(params, dict) and params.get("wait") == "true":
_log.debug("Watch timed out.")
raise etcdexcept.EtcdWatchTimedOut(
"Watch timed out: %r" % exc_obj,
cause=exc_obj
)
if self._allow_reconnect and fut._retry:
_log.info("Reconnection allowed, looking for another "
"server.")
self._base_url = fut._retry.pop(0)
fut_reconnect = func(self, path, method, params=params, timeout=timeout)
fut_reconnect._retry = fut._retry
self.ioloop.add_future(fut_reconnect, _callback)
return
else:
_log.debug("Reconnection disabled, giving up.")
raise etcdexcept.EtcdConnectionFailed(
"Connection to etcd failed due to %r" % exc_obj,
cause=exc_obj
)
else:
_log.exception("Unexpected request failure.")
result = fut.result()
self._check_cluster_id(result)
result = self._handle_server_response(fut.result())
new_future.set_result(result)
def _handle_connection(self, connection, address):
if self.ssl_options is not None:
assert ssl, "Python 2.6+ and OpenSSL required for SSL"
try:
connection = ssl_wrap_socket(connection,
self.ssl_options,
server_side=True,
do_handshake_on_connect=False)
except ssl.SSLError as err:
gen_log.exception(err)
if err.args[0] == ssl.SSL_ERROR_EOF:
return connection.close()
else:
raise
except socket.error as err:
# If the connection is closed immediately after it is created
# (as in a port scan), we can get one of several errors.
# wrap_socket makes an internal call to getpeername,
# which may return either EINVAL (Mac OS X) or ENOTCONN
# (Linux). If it returns ENOTCONN, this error is
# silently swallowed by the ssl module, so we need to
# catch another error later on (AttributeError in
# SSLSocketIOHandler._do_ssl_handshake).
# To test this behavior, try nmap with the -sT flag.
# https://github.com/facebook/tornado/pull/750
if err.args[0] in (errno.ECONNABORTED, errno.EINVAL):
return connection.close()
else:
raise
try:
if self.ssl_options is not None:
stream = SSLSocketIOHandler(connection, io_loop=self._io_loop)
else:
stream = SocketIOHandler(connection, io_loop=self._io_loop)
self.handle_stream(stream, address)
except Exception:
app_log.error("Error in connection callback", exc_info=True)
def _handle_connection(self, connection, address):
if self.ssl_options is not None:
assert ssl, "Python 2.6+ and OpenSSL required for SSL"
try:
connection = ssl_wrap_socket(connection,
self.ssl_options,
server_side=True,
do_handshake_on_connect=False)
except ssl.SSLError as err:
gen_log.exception(err)
if err.args[0] == ssl.SSL_ERROR_EOF:
return connection.close()
else:
raise
except socket.error as err:
# If the connection is closed immediately after it is created
# (as in a port scan), we can get one of several errors.
# wrap_socket makes an internal call to getpeername,
# which may return either EINVAL (Mac OS X) or ENOTCONN
# (Linux). If it returns ENOTCONN, this error is
# silently swallowed by the ssl module, so we need to
# catch another error later on (AttributeError in
# SSLSocketIOHandler._do_ssl_handshake).
# To test this behavior, try nmap with the -sT flag.
# https://github.com/facebook/tornado/pull/750
if err.args[0] in (errno.ECONNABORTED, errno.EINVAL):
return connection.close()
else:
raise
try:
if self.ssl_options is not None:
stream = SSLSocketIOHandler(connection, io_loop=self._io_loop)
else:
stream = SocketIOHandler(connection, io_loop=self._io_loop)
self.handle_stream(stream, address)
except Exception:
app_log.error("Error in connection callback", exc_info=True)
def run():
options.parse_command_line()
if options.config:
options.parse_config_file(options.config)
options.storage = os.path.abspath(options.storage)
if os.getuid() == 0 and options.user:
pw = pwd.getpwnam(options.user)
uid, gid = pw.pw_uid, pw.pw_gid
log.info("Changind user to %s [%s:%s]", options.user, uid, gid)
os.setgid(uid)
os.setuid(uid)
try:
if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)):
log.info('Creating new package storage directory: "%s"', options.storage)
os.makedirs(options.storage)
def on_interrupt(*args):
log.warning("Receiving interrupt signal. Application will be stopped.")
exit(errno.EINTR)
log.debug("Preparing signal handling")
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
signal.signal(sig, on_interrupt)
def handle_pdb(sig, frame):
import pdb
pdb.Pdb().set_trace(frame)
if options.debug:
signal.signal(signal.SIGUSR2, handle_pdb)
log.debug("Creating application instance")
app = create_app(
options.debug,
options.secret,
options.gzip,
)
log.debug("Creating IOLoop instance.")
io_loop = IOLoop.current()
io_loop.run_sync(lambda: init_db(options.database))
if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)):
os.makedirs(options.cache_dir)
Cache.CACHE_DIR = options.cache_dir
log.info("Init thread pool with %d threads", options.pool_size)
handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size)
AsyncHTTPClient.configure(None, max_clients=options.max_http_clients)
PYPIClient.configure(
options.pypi_server,
handlers.base.BaseHandler.THREAD_POOL
)
pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop)
io_loop.add_callback(PYPIClient.packages)
io_loop.add_callback(pypi_updater.start)
log.info("Starting server http://%s:%d/", options.address, options.port)
http_server = HTTPServer(app, xheaders=options.proxy_mode)
http_server.listen(options.port, address=options.address)
log.debug('Setting "%s" as storage', options.storage)
PackageFile.set_storage(options.storage)
log.debug("Starting main loop")
io_loop.start()
except Exception as e:
log.fatal("Exception on main loop:")
log.exception(e)
exit(1)
else:
exit(0)
def run():
options.parse_command_line()
if options.config:
options.parse_config_file(options.config)
options.storage = os.path.abspath(options.storage)
if os.getuid() == 0 and options.user:
pw = pwd.getpwnam(options.user)
uid, gid = pw.pw_uid, pw.pw_gid
log.info("Changind user to %s [%s:%s]", options.user, uid, gid)
os.setgid(uid)
os.setuid(uid)
try:
if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)):
log.info('Creating new package storage directory: "%s"', options.storage)
os.makedirs(options.storage)
def on_interrupt(*args):
log.warning("Receiving interrupt signal. Application will be stopped.")
exit(errno.EINTR)
log.debug("Preparing signal handling")
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
signal.signal(sig, on_interrupt)
def handle_pdb(sig, frame):
import pdb
pdb.Pdb().set_trace(frame)
if options.debug:
signal.signal(signal.SIGUSR2, handle_pdb)
log.debug("Creating application instance")
app = create_app(
options.debug,
options.secret,
options.gzip,
)
log.debug("Creating IOLoop instance.")
io_loop = IOLoop.current()
io_loop.run_sync(lambda: init_db(options.database))
if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)):
os.makedirs(options.cache_dir)
Cache.CACHE_DIR = options.cache_dir
log.info("Init thread pool with %d threads", options.pool_size)
handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size)
AsyncHTTPClient.configure(None, max_clients=options.max_http_clients)
proxy_url = URL(os.getenv('{0}_proxy'.format(options.pypi_server.scheme)))
if proxy_url:
log.debug("Configuring for proxy: %s", proxy_url)
AsyncHTTPClient.configure(
'tornado.curl_httpclient.CurlAsyncHTTPClient',
defaults={
'proxy_host': proxy_url.host,
'proxy_port': proxy_url.port,
'proxy_username': proxy_url.user,
'proxy_password': proxy_url.password,
}
)
PYPIClient.configure(
options.pypi_server,
handlers.base.BaseHandler.THREAD_POOL
)
if options.pypi_proxy:
pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop)
io_loop.add_callback(PYPIClient.packages)
io_loop.add_callback(pypi_updater.start)
log.info("Starting server http://%s:%d/", options.address, options.port)
http_server = HTTPServer(app, xheaders=options.proxy_mode)
http_server.listen(options.port, address=options.address)
log.debug('Setting "%s" as storage', options.storage)
PackageFile.set_storage(options.storage)
log.debug("Starting main loop")
io_loop.start()
except Exception as e:
log.fatal("Exception on main loop:")
log.exception(e)
exit(1)
else:
exit(0)
def _handle_exception(self, typ, value, tb):
gen_log.exception("dubbo connection error [%s] [%s] %s", typ, value,
tb)