def start(self): print( self._conf.get('prefix')) self.stop() self._mcnt = 0 self._socketId = '{0}-{1}'.format(self._conf['prefix'], uuid4()) print(self._socketId) self._sock = self._ctx.socket(zmq.DEALER) self._monitor = self._sock.get_monitor_socket() self._sock.setsockopt_string(zmq.IDENTITY, self._socketId) self._sock.setsockopt(zmq.LINGER,1) self._stream = zmqstream.ZMQStream(self._sock, self._loop) self._stream.on_recv_stream(self.on_data_recv) try: self._stream.connect(self._broker) self._liveness = HEARTBEAT_LIVENESS except Exception as ex: self.on_error(['error', ex]) gen_log.exception(ex) self._loop.spawn_callback(self.on_socket_event,self._monitor) # self.hbeater = ioloop.PeriodicCallback(self.beat, period, self._loop) # self.hbeater.start() self.on_start()
def wrapper(self, *args, **kwargs): sign = self.get_argument('sign') user_id = self.get_argument('user_id') remote_ip = self.request.remote_ip request_msg = 'remote_ip: %s, params:%s, body:%s' % \ (remote_ip, self.request.query_arguments, self.request.body) response = None checker = Checker(redis_cli, remote_ip, user_id, sign) try: checker.check() except Exception as e: checker.fail() response = RESP_AUTH_CHECK_ERROR gen_log.error('request_msg[%s], err[%s]', request_msg, e) else: checker.success() if not response: try: response = func(self, *args, **kwargs) except Exception as e: response = RESP_TOP_MONITOR_ERROR gen_log.exception('request_msg[%s] err[%s]', request_msg, e) self.write(response) log_msg = 'request_msg[%s] ret[%s]' % (request_msg, response) gen_log.info(log_msg) return response
def crawl(self): """ :return: """ log.info('Sending API requests...') while True: try: yield gen.sleep(self.delay) gen_log.info('Items in queue: %d' % len(self.to_process)) if len(self.to_process) < 1: gen_log.info( 'Campaign requests completed. API Errors: %s' % self.api_errors) break # Read News providers from Queue. source = self.to_process.pop() gen_log.info('Provider: %s Source: %s' % (settings.NEWS_API, source)) if self.campaign.query: # Build News API URL for search query. url_params = { 'q': source, 'language': DEFAULT_LANGUAGE, 'sortBy': settings.NEWS_API_SORT_ORDER, 'pageSize': settings.NEWS_PAGE_SIZE } news_url = '%s%s?%s' % (settings.NEWS_API_URL, 'everything', urlencode(url_params)) else: url_params = { 'sources': source, 'sortBy': settings.NEWS_API_SORT_ORDER, 'pageSize': settings.NEWS_PAGE_SIZE } if source == 'recode': del url_params['sortBy'] news_url = '%s%s?%s' % (settings.NEWS_API_URL, 'top-headlines', urlencode(url_params)) request = httpclient.HTTPRequest( url=news_url, headers=_get_auth_headers(settings.NEWS_API_KEY), connect_timeout=CONNECTION_TIMEOUT, request_timeout=REQUEST_TIMEOUT) self.backlog.fetch(request, callback=partial(self.handle_request, source)) except httpclient.HTTPError as exception: gen_log.exception('crawl() %r' % exception) self.api_errors += 1 if self.handle_errors(self.api_errors): break except IndexError: break gen_log.info('Waiting for active Provider requests to complete...') yield gen.sleep(settings.CAMPAIGN_LIMIT)
def shutdown(self, io_loop): for func in self.funcs[:]: try: func() except: LOG.exception('Failed to call %r', func) LOG.warn( 'Shutdown initiated, will force stop in %d seconds', self.max_wait ) self.stop_loop(io_loop, self.time_func() + self.max_wait)
def _callback(fut): exc_obj = fut.exception() if exc_obj: if isinstance(exc_obj, HTTPError) and exc_obj.code < 500: try: self._handle_server_response(exc_obj.response) except Exception, ex: new_future.set_exception(ex) return elif (isinstance(exc_obj, HTTPError) and exc_obj.code > 500) or isinstance( exc_obj, socket.error): _log.error("Request to server %s failed: %r", self._base_url, exc_obj) if isinstance(params, dict) and params.get("wait") == "true" \ and hasattr(exc_obj, 'code') and exc_obj.code == 599: _log.debug("Watch timed out.") exc = etcdexcept.EtcdWatchTimedOut( "Watch timed out: %r" % exc_obj, cause=exc_obj) new_future.set_exception(exc) return elif self._allow_reconnect and fut._retry: _log.info( "Reconnection allowed, looking for another " "server.") self._base_url = fut._retry.pop(0) fut_reconnect = func(self, path, method, params=params, timeout=timeout) fut_reconnect._retry = fut._retry self.ioloop.add_future(fut_reconnect, _callback) return else: _log.debug("Reconnection disabled, giving up.") exc = etcdexcept.EtcdConnectionFailed( "Connection to etcd failed due to %r" % exc_obj, cause=exc_obj) new_future.set_exception(exc) return else: _log.exception("Unexpected request failure.") exc = etcdexcept.EtcdException(exc_obj.message) new_future.set_exception(exc) return
async def get(self): urls = self.get_arguments('url') http_client = AsyncHTTPClient() for url in urls: try: gen_log.info(f'>>>>>准备请求url: {url}') response = await http_client.fetch(url) gen_log.info(f'>>>>>已获取请求结果') except Exception as e: msg = "Error: %s" % e gen_log.exception(msg) self.write(msg) else: gen_log.info(response.body) self.write(response.body)
def resolve_urls_from_domain(url): # 通过url获取域名下面的所有的ip try: parsed_uri = urlparse(url) domain = parsed_uri.hostname port = parsed_uri.port ips = get_ips_by_domain(domain) urls = [] for ip in ips: urls.append("http://%s:%s%s" % (ip, port, parsed_uri.path)) return urls except Exception as e: gen_log.exception(e) return [url]
def _callback(fut): exc = fut.exc_info() if exc: exc_obj = exc[1] if isinstance(exc_obj, HTTPError) and exc_obj.code < 500: result = self._handle_server_response(exc_obj.response) new_future.set_result(result) return elif (isinstance(exc_obj, HTTPError) and exc_obj.code > 500) or isinstance( exc_obj, socket.error): _log.error("Request to server %s failed: %r", self._base_url, exc_obj) if isinstance(params, dict) and params.get("wait") == "true": _log.debug("Watch timed out.") raise etcdexcept.EtcdWatchTimedOut( "Watch timed out: %r" % exc_obj, cause=exc_obj) if self._allow_reconnect and fut._retry: _log.info( "Reconnection allowed, looking for another " "server.") self._base_url = fut._retry.pop(0) fut_reconnect = func(self, path, method, params=params, timeout=timeout) fut_reconnect._retry = fut._retry self.ioloop.add_future(fut_reconnect, _callback) return else: _log.debug("Reconnection disabled, giving up.") raise etcdexcept.EtcdConnectionFailed( "Connection to etcd failed due to %r" % exc_obj, cause=exc_obj) else: _log.exception("Unexpected request failure.") result = fut.result() self._check_cluster_id(result) result = self._handle_server_response(fut.result()) new_future.set_result(result)
def _callback(fut): exc = fut.exc_info() if exc: exc_obj = exc[1] if isinstance(exc_obj, HTTPError) and exc_obj.code < 500: result = self._handle_server_response(exc_obj.response) new_future.set_result(result) return elif (isinstance(exc_obj, HTTPError) and exc_obj.code > 500) or isinstance(exc_obj, socket.error): _log.error("Request to server %s failed: %r", self._base_url, exc_obj) if isinstance(params, dict) and params.get("wait") == "true": _log.debug("Watch timed out.") raise etcdexcept.EtcdWatchTimedOut( "Watch timed out: %r" % exc_obj, cause=exc_obj ) if self._allow_reconnect and fut._retry: _log.info("Reconnection allowed, looking for another " "server.") self._base_url = fut._retry.pop(0) fut_reconnect = func(self, path, method, params=params, timeout=timeout) fut_reconnect._retry = fut._retry self.ioloop.add_future(fut_reconnect, _callback) return else: _log.debug("Reconnection disabled, giving up.") raise etcdexcept.EtcdConnectionFailed( "Connection to etcd failed due to %r" % exc_obj, cause=exc_obj ) else: _log.exception("Unexpected request failure.") result = fut.result() self._check_cluster_id(result) result = self._handle_server_response(fut.result()) new_future.set_result(result)
def _handle_connection(self, connection, address): if self.ssl_options is not None: assert ssl, "Python 2.6+ and OpenSSL required for SSL" try: connection = ssl_wrap_socket(connection, self.ssl_options, server_side=True, do_handshake_on_connect=False) except ssl.SSLError as err: gen_log.exception(err) if err.args[0] == ssl.SSL_ERROR_EOF: return connection.close() else: raise except socket.error as err: # If the connection is closed immediately after it is created # (as in a port scan), we can get one of several errors. # wrap_socket makes an internal call to getpeername, # which may return either EINVAL (Mac OS X) or ENOTCONN # (Linux). If it returns ENOTCONN, this error is # silently swallowed by the ssl module, so we need to # catch another error later on (AttributeError in # SSLSocketIOHandler._do_ssl_handshake). # To test this behavior, try nmap with the -sT flag. # https://github.com/facebook/tornado/pull/750 if err.args[0] in (errno.ECONNABORTED, errno.EINVAL): return connection.close() else: raise try: if self.ssl_options is not None: stream = SSLSocketIOHandler(connection, io_loop=self._io_loop) else: stream = SocketIOHandler(connection, io_loop=self._io_loop) self.handle_stream(stream, address) except Exception: app_log.error("Error in connection callback", exc_info=True)
def run(): options.parse_command_line() if options.config: options.parse_config_file(options.config) options.storage = os.path.abspath(options.storage) if os.getuid() == 0 and options.user: pw = pwd.getpwnam(options.user) uid, gid = pw.pw_uid, pw.pw_gid log.info("Changind user to %s [%s:%s]", options.user, uid, gid) os.setgid(uid) os.setuid(uid) try: if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)): log.info('Creating new package storage directory: "%s"', options.storage) os.makedirs(options.storage) def on_interrupt(*args): log.warning("Receiving interrupt signal. Application will be stopped.") exit(errno.EINTR) log.debug("Preparing signal handling") for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): signal.signal(sig, on_interrupt) def handle_pdb(sig, frame): import pdb pdb.Pdb().set_trace(frame) if options.debug: signal.signal(signal.SIGUSR2, handle_pdb) log.debug("Creating application instance") app = create_app( options.debug, options.secret, options.gzip, ) log.debug("Creating IOLoop instance.") io_loop = IOLoop.current() io_loop.run_sync(lambda: init_db(options.database)) if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)): os.makedirs(options.cache_dir) Cache.CACHE_DIR = options.cache_dir log.info("Init thread pool with %d threads", options.pool_size) handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size) AsyncHTTPClient.configure(None, max_clients=options.max_http_clients) PYPIClient.configure( options.pypi_server, handlers.base.BaseHandler.THREAD_POOL ) pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop) io_loop.add_callback(PYPIClient.packages) io_loop.add_callback(pypi_updater.start) log.info("Starting server http://%s:%d/", options.address, options.port) http_server = HTTPServer(app, xheaders=options.proxy_mode) http_server.listen(options.port, address=options.address) log.debug('Setting "%s" as storage', options.storage) PackageFile.set_storage(options.storage) log.debug("Starting main loop") io_loop.start() except Exception as e: log.fatal("Exception on main loop:") log.exception(e) exit(1) else: exit(0)
def run(): options.parse_command_line() if options.config: options.parse_config_file(options.config) options.storage = os.path.abspath(options.storage) if os.getuid() == 0 and options.user: pw = pwd.getpwnam(options.user) uid, gid = pw.pw_uid, pw.pw_gid log.info("Changind user to %s [%s:%s]", options.user, uid, gid) os.setgid(uid) os.setuid(uid) try: if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)): log.info('Creating new package storage directory: "%s"', options.storage) os.makedirs(options.storage) def on_interrupt(*args): log.warning("Receiving interrupt signal. Application will be stopped.") exit(errno.EINTR) log.debug("Preparing signal handling") for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): signal.signal(sig, on_interrupt) def handle_pdb(sig, frame): import pdb pdb.Pdb().set_trace(frame) if options.debug: signal.signal(signal.SIGUSR2, handle_pdb) log.debug("Creating application instance") app = create_app( options.debug, options.secret, options.gzip, ) log.debug("Creating IOLoop instance.") io_loop = IOLoop.current() io_loop.run_sync(lambda: init_db(options.database)) if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)): os.makedirs(options.cache_dir) Cache.CACHE_DIR = options.cache_dir log.info("Init thread pool with %d threads", options.pool_size) handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size) AsyncHTTPClient.configure(None, max_clients=options.max_http_clients) proxy_url = URL(os.getenv('{0}_proxy'.format(options.pypi_server.scheme))) if proxy_url: log.debug("Configuring for proxy: %s", proxy_url) AsyncHTTPClient.configure( 'tornado.curl_httpclient.CurlAsyncHTTPClient', defaults={ 'proxy_host': proxy_url.host, 'proxy_port': proxy_url.port, 'proxy_username': proxy_url.user, 'proxy_password': proxy_url.password, } ) PYPIClient.configure( options.pypi_server, handlers.base.BaseHandler.THREAD_POOL ) if options.pypi_proxy: pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop) io_loop.add_callback(PYPIClient.packages) io_loop.add_callback(pypi_updater.start) log.info("Starting server http://%s:%d/", options.address, options.port) http_server = HTTPServer(app, xheaders=options.proxy_mode) http_server.listen(options.port, address=options.address) log.debug('Setting "%s" as storage', options.storage) PackageFile.set_storage(options.storage) log.debug("Starting main loop") io_loop.start() except Exception as e: log.fatal("Exception on main loop:") log.exception(e) exit(1) else: exit(0)
def _handle_exception(self, typ, value, tb): gen_log.exception("dubbo connection error [%s] [%s] %s", typ, value, tb)