Exemple #1
0
    def start(self):

        print( self._conf.get('prefix'))

        self.stop()
        self._mcnt = 0
        self._socketId = '{0}-{1}'.format(self._conf['prefix'], uuid4())
        print(self._socketId)

        self._sock = self._ctx.socket(zmq.DEALER)
        self._monitor = self._sock.get_monitor_socket()
        self._sock.setsockopt_string(zmq.IDENTITY, self._socketId)
        self._sock.setsockopt(zmq.LINGER,1)
        self._stream = zmqstream.ZMQStream(self._sock, self._loop)
        self._stream.on_recv_stream(self.on_data_recv)

        try:
            self._stream.connect(self._broker)
            self._liveness = HEARTBEAT_LIVENESS
        except Exception as ex:
            self.on_error(['error', ex])
            gen_log.exception(ex)

        self._loop.spawn_callback(self.on_socket_event,self._monitor)
        # self.hbeater = ioloop.PeriodicCallback(self.beat, period, self._loop)
        # self.hbeater.start()
        self.on_start()
Exemple #2
0
    def wrapper(self, *args, **kwargs):
        sign = self.get_argument('sign')
        user_id = self.get_argument('user_id')
        remote_ip = self.request.remote_ip
        request_msg = 'remote_ip: %s, params:%s, body:%s' % \
                      (remote_ip, self.request.query_arguments, self.request.body)

        response = None
        checker = Checker(redis_cli, remote_ip, user_id, sign)
        try:
            checker.check()
        except Exception as e:
            checker.fail()
            response = RESP_AUTH_CHECK_ERROR
            gen_log.error('request_msg[%s], err[%s]', request_msg, e)
        else:
            checker.success()

        if not response:
            try:
                response = func(self, *args, **kwargs)
            except Exception as e:
                response = RESP_TOP_MONITOR_ERROR
                gen_log.exception('request_msg[%s] err[%s]', request_msg, e)
        self.write(response)

        log_msg = 'request_msg[%s] ret[%s]' % (request_msg, response)
        gen_log.info(log_msg)
        return response
Exemple #3
0
    def crawl(self):
        """

        :return:
        """
        log.info('Sending API requests...')
        while True:
            try:
                yield gen.sleep(self.delay)
                gen_log.info('Items in queue: %d' % len(self.to_process))
                if len(self.to_process) < 1:
                    gen_log.info(
                        'Campaign requests completed. API Errors: %s' %
                        self.api_errors)
                    break
                # Read News providers from Queue.
                source = self.to_process.pop()
                gen_log.info('Provider: %s Source: %s' %
                             (settings.NEWS_API, source))
                if self.campaign.query:
                    # Build News API URL for search query.
                    url_params = {
                        'q': source,
                        'language': DEFAULT_LANGUAGE,
                        'sortBy': settings.NEWS_API_SORT_ORDER,
                        'pageSize': settings.NEWS_PAGE_SIZE
                    }
                    news_url = '%s%s?%s' % (settings.NEWS_API_URL,
                                            'everything',
                                            urlencode(url_params))
                else:
                    url_params = {
                        'sources': source,
                        'sortBy': settings.NEWS_API_SORT_ORDER,
                        'pageSize': settings.NEWS_PAGE_SIZE
                    }
                    if source == 'recode':
                        del url_params['sortBy']
                    news_url = '%s%s?%s' % (settings.NEWS_API_URL,
                                            'top-headlines',
                                            urlencode(url_params))
                request = httpclient.HTTPRequest(
                    url=news_url,
                    headers=_get_auth_headers(settings.NEWS_API_KEY),
                    connect_timeout=CONNECTION_TIMEOUT,
                    request_timeout=REQUEST_TIMEOUT)
                self.backlog.fetch(request,
                                   callback=partial(self.handle_request,
                                                    source))

            except httpclient.HTTPError as exception:
                gen_log.exception('crawl() %r' % exception)
                self.api_errors += 1
                if self.handle_errors(self.api_errors):
                    break
            except IndexError:
                break
        gen_log.info('Waiting for active Provider requests to complete...')
        yield gen.sleep(settings.CAMPAIGN_LIMIT)
Exemple #4
0
    def shutdown(self, io_loop):
        for func in self.funcs[:]:
            try:
                func()
            except:
                LOG.exception('Failed to call %r', func)

        LOG.warn(
            'Shutdown initiated, will force stop in %d seconds',
            self.max_wait
        )
        self.stop_loop(io_loop, self.time_func() + self.max_wait)
Exemple #5
0
            def _callback(fut):
                exc_obj = fut.exception()
                if exc_obj:
                    if isinstance(exc_obj, HTTPError) and exc_obj.code < 500:
                        try:
                            self._handle_server_response(exc_obj.response)
                        except Exception, ex:
                            new_future.set_exception(ex)
                        return
                    elif (isinstance(exc_obj, HTTPError)
                          and exc_obj.code > 500) or isinstance(
                              exc_obj, socket.error):
                        _log.error("Request to server %s failed: %r",
                                   self._base_url, exc_obj)
                        if isinstance(params, dict) and params.get("wait") == "true" \
                                and hasattr(exc_obj, 'code') and exc_obj.code == 599:
                            _log.debug("Watch timed out.")
                            exc = etcdexcept.EtcdWatchTimedOut(
                                "Watch timed out: %r" % exc_obj, cause=exc_obj)
                            new_future.set_exception(exc)
                            return
                        elif self._allow_reconnect and fut._retry:
                            _log.info(
                                "Reconnection allowed, looking for another "
                                "server.")
                            self._base_url = fut._retry.pop(0)
                            fut_reconnect = func(self,
                                                 path,
                                                 method,
                                                 params=params,
                                                 timeout=timeout)
                            fut_reconnect._retry = fut._retry
                            self.ioloop.add_future(fut_reconnect, _callback)
                            return

                        else:
                            _log.debug("Reconnection disabled, giving up.")
                            exc = etcdexcept.EtcdConnectionFailed(
                                "Connection to etcd failed due to %r" %
                                exc_obj,
                                cause=exc_obj)
                            new_future.set_exception(exc)
                            return

                    else:
                        _log.exception("Unexpected request failure.")
                        exc = etcdexcept.EtcdException(exc_obj.message)
                        new_future.set_exception(exc)
                        return
Exemple #6
0
 async def get(self):
     urls = self.get_arguments('url')
     http_client = AsyncHTTPClient()
     for url in urls:
         try:
             gen_log.info(f'>>>>>准备请求url: {url}')
             response = await http_client.fetch(url)
             gen_log.info(f'>>>>>已获取请求结果')
         except Exception as e:
             msg = "Error: %s" % e
             gen_log.exception(msg)
             self.write(msg)
         else:
             gen_log.info(response.body)
             self.write(response.body)
Exemple #7
0
def resolve_urls_from_domain(url):
    # 通过url获取域名下面的所有的ip
    try:
        parsed_uri = urlparse(url)
        domain = parsed_uri.hostname
        port = parsed_uri.port
        ips = get_ips_by_domain(domain)
        urls = []

        for ip in ips:
            urls.append("http://%s:%s%s" % (ip, port, parsed_uri.path))
        return urls
    except Exception as e:
        gen_log.exception(e)
        return [url]
Exemple #8
0
            def _callback(fut):
                exc = fut.exc_info()
                if exc:
                    exc_obj = exc[1]
                    if isinstance(exc_obj, HTTPError) and exc_obj.code < 500:
                        result = self._handle_server_response(exc_obj.response)
                        new_future.set_result(result)
                        return
                    elif (isinstance(exc_obj, HTTPError)
                          and exc_obj.code > 500) or isinstance(
                              exc_obj, socket.error):
                        _log.error("Request to server %s failed: %r",
                                   self._base_url, exc_obj)
                        if isinstance(params,
                                      dict) and params.get("wait") == "true":
                            _log.debug("Watch timed out.")
                            raise etcdexcept.EtcdWatchTimedOut(
                                "Watch timed out: %r" % exc_obj, cause=exc_obj)
                        if self._allow_reconnect and fut._retry:
                            _log.info(
                                "Reconnection allowed, looking for another "
                                "server.")
                            self._base_url = fut._retry.pop(0)
                            fut_reconnect = func(self,
                                                 path,
                                                 method,
                                                 params=params,
                                                 timeout=timeout)
                            fut_reconnect._retry = fut._retry
                            self.ioloop.add_future(fut_reconnect, _callback)
                            return

                        else:
                            _log.debug("Reconnection disabled, giving up.")
                            raise etcdexcept.EtcdConnectionFailed(
                                "Connection to etcd failed due to %r" %
                                exc_obj,
                                cause=exc_obj)

                    else:
                        _log.exception("Unexpected request failure.")

                result = fut.result()
                self._check_cluster_id(result)
                result = self._handle_server_response(fut.result())
                new_future.set_result(result)
Exemple #9
0
            def _callback(fut):
                exc = fut.exc_info()
                if exc:
                    exc_obj = exc[1]
                    if isinstance(exc_obj, HTTPError) and exc_obj.code < 500:
                        result = self._handle_server_response(exc_obj.response)
                        new_future.set_result(result)
                        return
                    elif (isinstance(exc_obj, HTTPError) and exc_obj.code > 500) or isinstance(exc_obj, socket.error):
                        _log.error("Request to server %s failed: %r",
                                   self._base_url, exc_obj)
                        if isinstance(params, dict) and params.get("wait") == "true":
                            _log.debug("Watch timed out.")
                            raise etcdexcept.EtcdWatchTimedOut(
                                    "Watch timed out: %r" % exc_obj,
                                    cause=exc_obj
                            )
                        if self._allow_reconnect and fut._retry:
                            _log.info("Reconnection allowed, looking for another "
                                      "server.")
                            self._base_url = fut._retry.pop(0)
                            fut_reconnect = func(self, path, method, params=params, timeout=timeout)
                            fut_reconnect._retry = fut._retry
                            self.ioloop.add_future(fut_reconnect, _callback)
                            return

                        else:
                            _log.debug("Reconnection disabled, giving up.")
                            raise etcdexcept.EtcdConnectionFailed(
                                    "Connection to etcd failed due to %r" % exc_obj,
                                    cause=exc_obj
                            )

                    else:
                        _log.exception("Unexpected request failure.")

                result = fut.result()
                self._check_cluster_id(result)
                result = self._handle_server_response(fut.result())
                new_future.set_result(result)
Exemple #10
0
    def _handle_connection(self, connection, address):
        if self.ssl_options is not None:
            assert ssl, "Python 2.6+ and OpenSSL required for SSL"
            try:
                connection = ssl_wrap_socket(connection,
                                             self.ssl_options,
                                             server_side=True,
                                             do_handshake_on_connect=False)
            except ssl.SSLError as err:
                gen_log.exception(err)

                if err.args[0] == ssl.SSL_ERROR_EOF:
                    return connection.close()
                else:
                    raise
            except socket.error as err:
                # If the connection is closed immediately after it is created
                # (as in a port scan), we can get one of several errors.
                # wrap_socket makes an internal call to getpeername,
                # which may return either EINVAL (Mac OS X) or ENOTCONN
                # (Linux). If it returns ENOTCONN, this error is
                # silently swallowed by the ssl module, so we need to
                # catch another error later on (AttributeError in
                # SSLSocketIOHandler._do_ssl_handshake).
                # To test this behavior, try nmap with the -sT flag.
                # https://github.com/facebook/tornado/pull/750
                if err.args[0] in (errno.ECONNABORTED, errno.EINVAL):
                    return connection.close()
                else:
                    raise
        try:
            if self.ssl_options is not None:
                stream = SSLSocketIOHandler(connection, io_loop=self._io_loop)
            else:
                stream = SocketIOHandler(connection, io_loop=self._io_loop)
            self.handle_stream(stream, address)
        except Exception:
            app_log.error("Error in connection callback", exc_info=True)
Exemple #11
0
    def _handle_connection(self, connection, address):
        if self.ssl_options is not None:
            assert ssl, "Python 2.6+ and OpenSSL required for SSL"
            try:
                connection = ssl_wrap_socket(connection,
                                             self.ssl_options,
                                             server_side=True,
                                             do_handshake_on_connect=False)
            except ssl.SSLError as err:
                gen_log.exception(err)

                if err.args[0] == ssl.SSL_ERROR_EOF:
                    return connection.close()
                else:
                    raise
            except socket.error as err:
                # If the connection is closed immediately after it is created
                # (as in a port scan), we can get one of several errors.
                # wrap_socket makes an internal call to getpeername,
                # which may return either EINVAL (Mac OS X) or ENOTCONN
                # (Linux). If it returns ENOTCONN, this error is
                # silently swallowed by the ssl module, so we need to
                # catch another error later on (AttributeError in
                # SSLSocketIOHandler._do_ssl_handshake).
                # To test this behavior, try nmap with the -sT flag.
                # https://github.com/facebook/tornado/pull/750
                if err.args[0] in (errno.ECONNABORTED, errno.EINVAL):
                    return connection.close()
                else:
                    raise
        try:
            if self.ssl_options is not None:
                stream = SSLSocketIOHandler(connection, io_loop=self._io_loop)
            else:
                stream = SocketIOHandler(connection, io_loop=self._io_loop)
            self.handle_stream(stream, address)
        except Exception:
            app_log.error("Error in connection callback", exc_info=True)
Exemple #12
0
def run():
    options.parse_command_line()

    if options.config:
        options.parse_config_file(options.config)

    options.storage = os.path.abspath(options.storage)

    if os.getuid() == 0 and options.user:
        pw = pwd.getpwnam(options.user)
        uid, gid = pw.pw_uid, pw.pw_gid
        log.info("Changind user to %s [%s:%s]", options.user, uid, gid)
        os.setgid(uid)
        os.setuid(uid)

    try:
        if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)):
            log.info('Creating new package storage directory: "%s"', options.storage)
            os.makedirs(options.storage)

        def on_interrupt(*args):
            log.warning("Receiving interrupt signal. Application will be stopped.")
            exit(errno.EINTR)

        log.debug("Preparing signal handling")
        for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
            signal.signal(sig, on_interrupt)

        def handle_pdb(sig, frame):
            import pdb
            pdb.Pdb().set_trace(frame)

        if options.debug:
            signal.signal(signal.SIGUSR2, handle_pdb)

        log.debug("Creating application instance")
        app = create_app(
            options.debug,
            options.secret,
            options.gzip,
        )

        log.debug("Creating IOLoop instance.")
        io_loop = IOLoop.current()

        io_loop.run_sync(lambda: init_db(options.database))

        if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)):
            os.makedirs(options.cache_dir)

        Cache.CACHE_DIR = options.cache_dir

        log.info("Init thread pool with %d threads", options.pool_size)
        handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size)

        AsyncHTTPClient.configure(None, max_clients=options.max_http_clients)

        PYPIClient.configure(
            options.pypi_server,
            handlers.base.BaseHandler.THREAD_POOL
        )

        pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop)

        io_loop.add_callback(PYPIClient.packages)
        io_loop.add_callback(pypi_updater.start)

        log.info("Starting server http://%s:%d/", options.address, options.port)
        http_server = HTTPServer(app, xheaders=options.proxy_mode)
        http_server.listen(options.port, address=options.address)

        log.debug('Setting "%s" as storage', options.storage)
        PackageFile.set_storage(options.storage)

        log.debug("Starting main loop")
        io_loop.start()
    except Exception as e:
        log.fatal("Exception on main loop:")
        log.exception(e)
        exit(1)
    else:
        exit(0)
Exemple #13
0
def run():
    options.parse_command_line()

    if options.config:
        options.parse_config_file(options.config)

    options.storage = os.path.abspath(options.storage)

    if os.getuid() == 0 and options.user:
        pw = pwd.getpwnam(options.user)
        uid, gid = pw.pw_uid, pw.pw_gid
        log.info("Changind user to %s [%s:%s]", options.user, uid, gid)
        os.setgid(uid)
        os.setuid(uid)

    try:
        if not all(f(options.storage) for f in (os.path.exists, os.path.isdir)):
            log.info('Creating new package storage directory: "%s"', options.storage)
            os.makedirs(options.storage)

        def on_interrupt(*args):
            log.warning("Receiving interrupt signal. Application will be stopped.")
            exit(errno.EINTR)

        log.debug("Preparing signal handling")
        for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
            signal.signal(sig, on_interrupt)

        def handle_pdb(sig, frame):
            import pdb
            pdb.Pdb().set_trace(frame)

        if options.debug:
            signal.signal(signal.SIGUSR2, handle_pdb)

        log.debug("Creating application instance")
        app = create_app(
            options.debug,
            options.secret,
            options.gzip,
        )

        log.debug("Creating IOLoop instance.")
        io_loop = IOLoop.current()

        io_loop.run_sync(lambda: init_db(options.database))

        if not (os.path.exists(options.cache_dir) and os.path.isdir(options.cache_dir)):
            os.makedirs(options.cache_dir)

        Cache.CACHE_DIR = options.cache_dir

        log.info("Init thread pool with %d threads", options.pool_size)
        handlers.base.BaseHandler.THREAD_POOL = futures.ThreadPoolExecutor(options.pool_size)

        AsyncHTTPClient.configure(None, max_clients=options.max_http_clients)

        proxy_url = URL(os.getenv('{0}_proxy'.format(options.pypi_server.scheme)))
        if proxy_url:
            log.debug("Configuring for proxy: %s", proxy_url)
            AsyncHTTPClient.configure(
                    'tornado.curl_httpclient.CurlAsyncHTTPClient',
                    defaults={
                        'proxy_host': proxy_url.host,
                        'proxy_port': proxy_url.port,
                        'proxy_username': proxy_url.user,
                        'proxy_password': proxy_url.password,
                        }
                    )

        PYPIClient.configure(
            options.pypi_server,
            handlers.base.BaseHandler.THREAD_POOL
        )

        if options.pypi_proxy:
            pypi_updater = PeriodicCallback(PYPIClient.packages, HOUR * 1000, io_loop)

            io_loop.add_callback(PYPIClient.packages)
            io_loop.add_callback(pypi_updater.start)

        log.info("Starting server http://%s:%d/", options.address, options.port)
        http_server = HTTPServer(app, xheaders=options.proxy_mode)
        http_server.listen(options.port, address=options.address)

        log.debug('Setting "%s" as storage', options.storage)
        PackageFile.set_storage(options.storage)

        log.debug("Starting main loop")
        io_loop.start()
    except Exception as e:
        log.fatal("Exception on main loop:")
        log.exception(e)
        exit(1)
    else:
        exit(0)
Exemple #14
0
 def _handle_exception(self, typ, value, tb):
     gen_log.exception("dubbo connection error [%s] [%s] %s", typ, value,
                       tb)