def test_route_connection_refused_error(self): self.agent_mock.request.side_effect = MockAssist( [self._raise_connection_refused_error]) router_data = dict(node_id="http://somewhere", uaid=dummy_uaid) self.router_mock.clear_node.return_value = None log.addObserver(self._mockObserver) d = self.router.route_notification(self.notif, router_data) def verify_retry(fail): exc = fail.value ok_(exc, RouterException) eq_(exc.status_code, 503) eq_(len(self.router_mock.clear_node.mock_calls), 1) self.router_mock.clear_node.reset_mock() self.flushLoggedErrors() def verify_deliver(fail): ok_(self._contains_err('ConnectionRefusedError')) exc = fail.value ok_(exc, RouterException) eq_(exc.status_code, 503) eq_(len(self.router_mock.clear_node.mock_calls), 1) self.router_mock.clear_node.reset_mock() d = self.router.route_notification(self.notif, router_data) d.addBoth(verify_retry) return d d.addBoth(verify_deliver) return d
def observe(sentry_dsn): ''' Decorator adding twisted raven client and blocking client if reactor is not yet running. :param str sentry_dsn: URL of sentry API ''' # create blocking client: raven_client = raven.base.Client(sentry_dsn) # add twisted logObserver with twisted raven client: observer = get_observer(sentry_dsn) if observer: log.addObserver(observer) import pytest; pytest.set_trace() def decorator(function): @wraps(function) def wrapper(*args, **kwargs): ''' Calls original function, catches any exception, sends it to sentry and re-raises it again. ''' try: return function(*args, **kwargs) except: raven_client.captureException(sys.exc_info()) raise # re-raise caught exception return wrapper return decorator
def main(): factory = VatsimClientFactory() log.startLogging(sys.stdout) #log.addObserver(log.FileLogObserver(open("trace.log",'w'))) addObserver(FileLogObserver(open("trace.log",'w')).emit) reactor.connectTCP('USA-E.vatsim.net',6809, factory) reactor.run()
def start_filelog(filename=None): if filename is None: filename = "%s.proxy_log.txt" % datetime.now( ).strftime("%Y.%m.%d_%H.%M.%S") f = open(filename, "w") log.addObserver(MinecraftLogObserver(f).emit) msg("Started logging to file %s" % filename)
def __init__(self): class logObserver: def __init__(self, con): self.con = con def emit(self, eventDict): edm = eventDict['message'] if not edm: if eventDict['isError'] and 'failure' in eventDict: text = ((eventDict.get('why') or 'Unhandled Error') + '\n' + eventDict['failure'].getTraceback()) elif 'format' in eventDict: text = eventDict['format'] % eventDict else: text = str(eventDict) else: text = ' '.join(map(reflect.safe_str, edm)) self.con.addLine(text) stdscr = curses.initscr() # initialize curses self.screen = Screen(stdscr) # create Screen object log.addObserver(logObserver(self.screen).emit) stdscr.refresh() reactor.addReader(self.screen) # add screen object as a reader to the reactor task.LoopingCall(self.screen.updateDisplay).start(.25)
def run(self): retn = False try: for user in self._authBlacklist: self._userdb.blacklistUser(user) site = self._site() if not site: return False # start listen for incoming request self.__tcpPort = reactor.listenTCP(self._port, site, self._connections, self._listenAddress) # setup signal handler self.__sighup = False signal.signal(signal.SIGHUP, self._sighupHandler) task.LoopingCall(self._reloadTask).start(60, False) # start processing Logging.info("start listening") log.addObserver(logSC3) reactor.run() retn = True except Exception, e: Logging.error(str(e))
def main(): config = ConfigParser.ConfigParser() config.read(["/etc/awsdns.ini", os.path.abspath("./awsdns.ini"), os.path.expanduser("~/awsnds.ini")]) # TODO: what happens if we use more than one DNS server? resolver = EC2Resolver( config, servers=[(config.get('awsdns', 'dns_server'), 53)] ) f = server.DNSServerFactory(clients=[resolver]) p = dns.DNSDatagramProtocol(f) reactor.listenUDP(53, p) reactor.listenTCP(53, f) try: loglevel = util.logging_constant(config.get('awsdns', 'loglevel')) except ConfigParser.NoOptionError: loglevel = logging.INFO try: logfile = config.get('awsdns', 'logfile') fh = open("awsdns.log", "a") except ConfigParser.NoOptionError: fh = sys.stdout observer = LevelFileLogObserver(fh, loglevel) log.addObserver(observer) reactor.run()
def main(self): """Parse arguments and run the script's main function via ``react``.""" observer = None try: if not self.log_directory.exists(): self.log_directory.makedirs() log_path = self.log_directory.child( b"%s-%d.log" % (os.path.basename(self.sys_module.argv[0]), os.getpid())) log_file = log_path.open("a") observer = FileLogObserver(log_file).emit addObserver(observer) msg("Arguments: %s" % (self.sys_module.argv,)) except (OSError, IOError): pass options = self._parse_options(self.sys_module.argv[1:]) # XXX: We shouldn't be using this private _reactor API. See # https://twistedmatrix.com/trac/ticket/6200 and # https://twistedmatrix.com/trac/ticket/7527 self._react(self.script.main, (options,), _reactor=self._reactor) # Not strictly necessary, but nice cleanup for tests: if observer is not None: removeObserver(observer) log_file.close()
def __init__(self, bot): self.clients = [] self.server = None self.bot = bot try: if "websocket" in bot.config and bot.config["websocket"]["enabled"] == "1": # Initialize twisted logging try: from twisted.python import log as twisted_log twisted_log.addObserver(WebSocketManager.on_log_message) except ImportError: log.error("twisted is not installed, websocket cannot be initialized.") return except: log.exception("Uncaught exception") return port = int(bot.config["websocket"]["port"]) secure = False key_path = None crt_path = None if "ssl" in bot.config["websocket"] and bot.config["websocket"]["ssl"] == "1": secure = True if "key_path" in bot.config["websocket"]: key_path = bot.config["websocket"]["key_path"] if "crt_path" in bot.config["websocket"]: crt_path = bot.config["websocket"]["crt_path"] self.server = WebSocketServer(self, port, secure, key_path, crt_path) except: log.exception("Uncaught exception in WebSocketManager")
def setUp(self): self.catcher = [] observer = self.catcher.append log.addObserver(observer) self.addCleanup(log.removeObserver, observer) self.db = MagicMock() self.ws = MagicMock()
def __init__(self, bot): self.clients = [] self.server = None self.bot = bot try: if 'websocket' in bot.config and bot.config['websocket']['enabled'] == '1': # Initialize twisted logging try: from twisted.python import log as twisted_log twisted_log.addObserver(WebSocketManager.on_log_message) except ImportError: log.error('twisted is not installed, websocket cannot be initialized.') return except: log.exception('Uncaught exception') return port = int(bot.config['websocket']['port']) secure = False key_path = None crt_path = None if 'ssl' in bot.config['websocket'] and bot.config['websocket']['ssl'] == '1': secure = True if 'key_path' in bot.config['websocket']: key_path = bot.config['websocket']['key_path'] if 'crt_path' in bot.config['websocket']: crt_path = bot.config['websocket']['crt_path'] self.server = WebSocketServer(self, port, secure, key_path, crt_path) except: log.exception('Uncaught exception in WebSocketManager')
def test_connectionLostLogMsg(self): """ When a connection is lost, an informative message should be logged (see L{getExpectedConnectionLostLogMsg}): an address identifying the port and the fact that it was closed. """ loggedMessages = [] def logConnectionLostMsg(eventDict): loggedMessages.append(log.textFromEventDict(eventDict)) reactor = self.buildReactor() p = self.getListeningPort(reactor) expectedMessage = self.getExpectedConnectionLostLogMsg(p) log.addObserver(logConnectionLostMsg) def stopReactor(ignored): log.removeObserver(logConnectionLostMsg) reactor.stop() def doStopListening(): log.addObserver(logConnectionLostMsg) maybeDeferred(p.stopListening).addCallback(stopReactor) reactor.callWhenRunning(doStopListening) reactor.run() self.assertIn(expectedMessage, loggedMessages)
def __init__(self, **kwargs): ''' Initialize the C{Scheduler} instance. ''' # Handle command line options from optparse import OptionParser parser = OptionParser(usage="%prog") parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=False, help='Enable verbose mode') (options, args) = parser.parse_args() # Don't accept any positional arguments if args: parser.error('Positional parameters are not supported: %s' % ' '.join(args)) sys.exit(1) # Logging # TODO: log in database table perhaps? See ticket:6. log_fp = open(os.path.join(ssscrape.config.LOG_DIR, 'ssscrape-scheduler.log'), 'a') log.addObserver(log.FileLogObserver(log_fp).emit) if options.verbose: log.addObserver(log.FileLogObserver(sys.stdout).emit) # We cannot recover from SQL errors. ssscrape.database.add_error_callback(self.stop)
def test_malformedHeaderCGI(self): """ Check for the error message in the duplicated header """ cgiFilename = self.writeCGI(BROKEN_HEADER_CGI) portnum = self.startServer(cgiFilename) url = "http://localhost:%d/cgi" % (portnum,) url = url.encode("ascii") agent = client.Agent(reactor) d = agent.request(b"GET", url) d.addCallback(discardBody) loggedMessages = [] def addMessage(eventDict): loggedMessages.append(log.textFromEventDict(eventDict)) log.addObserver(addMessage) self.addCleanup(log.removeObserver, addMessage) def checkResponse(ignored): self.assertIn("ignoring malformed CGI header: " + repr(b'XYZ'), loggedMessages) d.addCallback(checkResponse) return d
def test_startLoggingTwice(self): """ There are some obscure error conditions that can occur when logging is started twice. See http://twistedmatrix.com/trac/ticket/3289 for more information. """ self._startLoggingCleanup() # The bug is particular to the way that the t.p.log 'global' function # handle stdout. If we use our own stream, the error doesn't occur. If # we use our own LogPublisher, the error doesn't occur. sys.stdout = StringIO() def showError(eventDict): if eventDict['isError']: sys.__stdout__.write(eventDict['failure'].getTraceback()) log.addObserver(showError) self.addCleanup(log.removeObserver, showError) observer = log.startLogging(sys.stdout) self.addCleanup(observer.stop) # At this point, we expect that sys.stdout is a StdioOnnaStick object. self.assertIsInstance(sys.stdout, LoggingFile) fakeStdout = sys.stdout observer = log.startLogging(sys.stdout) self.assertIs(sys.stdout, fakeStdout)
def test_send_catch_log(self): test_signal = object() handlers_called = set() def log_received(event): handlers_called.add(log_received) assert "error_handler" in event['message'][0] assert event['logLevel'] == log.ERROR txlog.addObserver(log_received) dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) result = yield defer.maybeDeferred(self._get_result, test_signal, arg='test', handlers_called=handlers_called) assert self.error_handler in handlers_called assert self.ok_handler in handlers_called assert log_received in handlers_called self.assertEqual(result[0][0], self.error_handler) self.assert_(isinstance(result[0][1], Failure)) self.assertEqual(result[1], (self.ok_handler, "OK")) txlog.removeObserver(log_received) self.flushLoggedErrors() dispatcher.disconnect(self.error_handler, signal=test_signal) dispatcher.disconnect(self.ok_handler, signal=test_signal)
def test_deferred(self): stdout = StringIO() log.addObserver(LevelFileLogObserver(stdout, logging.INFO)) self.call_count = 0 def test_result(d, count): long_calculation(count) self.call_count += 1 @twisted_timeit def defer_calc(count_iteration, count_run): d = defer.Deferred() for i in xrange(count_iteration): d.addCallback(test_result, count_run) return d d = defer_calc(self.count_iteration, self.count_run) d.callback(None) self.assertEqual(self.call_count, self.count_iteration) result_log = stdout.getvalue() self.assertEqual('Time for deferred' in result_log, True) print '\n', self.sep_line_two, '\nLog from test_deferred:\n', \ self.sep_line_one, '\n', result_log, '\n', self.sep_line_two, '\n'
def test_wsgiErrors(self): """ The C{'wsgi.errors'} key of the C{environ} C{dict} passed to the application is a file-like object (as defined in the U{Input and Errors Streams<http://www.python.org/dev/peps/pep-0333/#input-and-error-streams>} section of PEP 333) which converts bytes written to it into events for the logging system. """ events = [] addObserver(events.append) self.addCleanup(removeObserver, events.append) errors = self.render('GET', '1.1', [], ['']) def cbErrors((environ, startApplication)): errors = environ['wsgi.errors'] errors.write('some message\n') errors.writelines(['another\nmessage\n']) errors.flush() self.assertEqual(events[0]['message'], ('some message\n',)) self.assertEqual(events[0]['system'], 'wsgi') self.assertTrue(events[0]['isError']) self.assertEqual(events[1]['message'], ('another\nmessage\n',)) self.assertEqual(events[1]['system'], 'wsgi') self.assertTrue(events[1]['isError']) self.assertEqual(len(events), 2) errors.addCallback(cbErrors) return errors
def assertLogMessage(testCase, expectedMessages, callable, *args, **kwargs): """ Assert that the callable logs the expected messages when called. XXX: Put this somewhere where it can be re-used elsewhere. See #6677. @param testCase: The test case controlling the test which triggers the logged messages and on which assertions will be called. @type testCase: L{unittest.SynchronousTestCase} @param expectedMessages: A L{list} of the expected log messages @type expectedMessages: L{list} @param callable: The function which is expected to produce the C{expectedMessages} when called. @type callable: L{callable} @param args: Positional arguments to be passed to C{callable}. @type args: L{list} @param kwargs: Keyword arguments to be passed to C{callable}. @type kwargs: L{dict} """ loggedMessages = [] log.addObserver(loggedMessages.append) testCase.addCleanup(log.removeObserver, loggedMessages.append) callable(*args, **kwargs) testCase.assertEqual( [m['message'][0] for m in loggedMessages], expectedMessages)
def __init__(self, observer): tw_log.addObserver(observer) for k, v in self.levels.items(): method_name = v.lower() setattr(self, method_name, self._genLogger(k))
def main(): parser = create_parser() args = parser.parse_args() if args.debug or (not args.logfile and not args.db): print 'debug logging to console' log.addObserver(log.FileLogObserver(sys.stdout).emit) if not args.logfile and args.db: con = sqlite3.connect(args.db) try: # checking if db exists con.execute('select count(*) from revisions') except sqlite3.OperationalError: print 'creating db ' + args.db create_db(args.db) con = sqlite3.connect(args.db) print 'logging to ' + args.db log.addObserver(partial(db_observer, connection=con)) if isinstance(args.logfile, basestring): log_file = open(args.logfile, 'a') print 'logging to ' + str(log_file) log.startLogging(log_file) factory = WebSocketClientFactory(args.websocket) factory.protocol = RecordClientProtocol connectWS(factory) reactor.run()
def setUp(self): self.catcher = [] observer = self.catcher.append log.addObserver(observer) self.addCleanup(log.removeObserver, observer) self.node = Node(digest("test"), "127.0.0.1", 1234) self.router = RoutingTable(self, 20, self.node.id)
def setUp(self): """ Create a temporary file with a fixed payload of 64 bytes. Create a resource for that file and create a request which will be for that resource. Each test can set a different range header to test different aspects of the implementation. """ path = FilePath(self.mktemp()) # This is just a jumble of random stuff. It's supposed to be a good # set of data for this test, particularly in order to avoid # accidentally seeing the right result by having a byte sequence # repeated at different locations or by having byte values which are # somehow correlated with their position in the string. self.payload = ('\xf8u\xf3E\x8c7\xce\x00\x9e\xb6a0y0S\xf0\xef\xac\xb7' '\xbe\xb5\x17M\x1e\x136k{\x1e\xbe\x0c\x07\x07\t\xd0' '\xbckY\xf5I\x0b\xb8\x88oZ\x1d\x85b\x1a\xcdk\xf2\x1d' '&\xfd%\xdd\x82q/A\x10Y\x8b') path.setContent(self.payload) self.file = path.open() self.resource = static.File(self.file.name) self.resource.isLeaf = 1 self.request = DummyRequest(['']) self.request.uri = self.file.name self.catcher = [] log.addObserver(self.catcher.append)
def main(port, database=None, debug=False, stdout=True): if stdout: log.startLogging(sys.stdout) if debug: log.addObserver(shutdownOnError) reactor.listenTCP(port, ServerFactory(database)) reactor.run()
def _test_level(self, level, messages_number): observer = LevelFileLogObserver(self.log_file, level) log.addObserver(observer) self.LOG.debug('test debug') self.LOG.info('test info') self.LOG.warning('test warning') self.LOG.error('test error') self.LOG.critical('test critical') log.removeObserver(observer) with open(self.log_path) as f: lines = [line.strip() for line in f.readlines()] self.assertEqual(len(lines), messages_number) for line in lines: m = re.match(self.rx, line) self.assertIsNotNone(m) time, level_name, system, entry_text = m.groups() time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S") entry_level = logging.getLevelName(level_name) self.assertGreaterEqual(entry_level, level) self.assertEqual(system, __name__) text = "test {0}".format(level_name.lower()) self.assertEqual(entry_text, text)
def stopOnError(case, reactor, publisher=None): """ Stop the reactor as soon as any error is logged on the given publisher. This is beneficial for tests which will wait for a L{Deferred} to fire before completing (by passing or failing). Certain implementation bugs may prevent the L{Deferred} from firing with any result at all (consider a protocol's {dataReceived} method that raises an exception: this exception is logged but it won't ever cause a L{Deferred} to fire). In that case the test would have to complete by timing out which is a much less desirable outcome than completing as soon as the unexpected error is encountered. @param case: A L{SynchronousTestCase} to use to clean up the necessary log observer when the test is over. @param reactor: The reactor to stop. @param publisher: A L{LogPublisher} to watch for errors. If C{None}, the global log publisher will be watched. """ if publisher is None: from twisted.python import log as publisher running = [None] def stopIfError(event): if running and event.get('isError'): running.pop() reactor.stop() publisher.addObserver(stopIfError) case.addCleanup(publisher.removeObserver, stopIfError)
def __init__(self, crawler): self.crawler = crawler self.timeout = settings.getint('CLOSESPIDER_TIMEOUT') self.itemcount = settings.getint('CLOSESPIDER_ITEMCOUNT') # XXX: legacy support - remove for future releases if settings.getint('CLOSESPIDER_ITEMPASSED'): warnings.warn("CLOSESPIDER_ITEMPASSED setting is deprecated, use CLOSESPIDER_ITEMCOUNT instead", ScrapyDeprecationWarning) self.pagecount = settings.getint('CLOSESPIDER_ITEMPASSED') self.pagecount = settings.getint('CLOSESPIDER_PAGECOUNT') self.errorcount = settings.getint('CLOSESPIDER_ERRORCOUNT') self.errorcounts = defaultdict(int) self.pagecounts = defaultdict(int) self.counts = defaultdict(int) self.tasks = {} if self.errorcount: txlog.addObserver(self.catch_log) if self.pagecount: dispatcher.connect(self.page_count, signal=signals.response_received) if self.timeout: dispatcher.connect(self.spider_opened, signal=signals.spider_opened) if self.itemcount: dispatcher.connect(self.item_scraped, signal=signals.item_scraped) dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
def test_default_item_completed(self): item = dict(name='name') assert self.pipe.item_completed([], item, self.info) is item # Check that failures are logged by default fail = Failure(Exception()) results = [(True, 1), (False, fail)] events = [] txlog.addObserver(events.append) new_item = self.pipe.item_completed(results, item, self.info) txlog.removeObserver(events.append) self.flushLoggedErrors() assert new_item is item assert len(events) == 1 assert events[0]['logLevel'] == log.ERROR assert events[0]['failure'] is fail # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False events = [] txlog.addObserver(events.append) new_item = self.pipe.item_completed(results, item, self.info) txlog.removeObserver(events.append) self.flushLoggedErrors() assert new_item is item assert len(events) == 0
def startLogging(self, observer): """ Override C{startLogging} to call L{log.addObserver} instead of L{log.startLoggingWithObserver}. """ log.addObserver(observer) self.logger._initialLog()
def test_newPluginsOnReadOnlyPath(self): """ Verify that a failure to write the dropin.cache file on a read-only path will not affect the list of plugins returned. Note: this test should pass on both Linux and Windows, but may not provide useful coverage on Windows due to the different meaning of "read-only directory". """ self.unlockSystem() self.sysplug.child('newstuff.py').setContent(pluginFileContents('one')) self.lockSystem() # Take the developer path out, so that the system plugins are actually # examined. sys.path.remove(self.devPath.path) # Start observing log events to see the warning events = [] addObserver(events.append) self.addCleanup(removeObserver, events.append) self.assertIn('one', self.getAllPlugins()) # Make sure something was logged about the cache. expected = "Unable to write to plugin cache %s: error number %d" % ( self.syscache.path, errno.EPERM) for event in events: if expected in textFromEventDict(event): break else: self.fail( "Did not observe unwriteable cache warning in log " "events: %r" % (events,))
def test_outputReceivedCompleteLineInvalidUTF8(self): """ Getting invalid UTF-8 results in the repr of the raw message """ events = [] self.addCleanup(log.removeObserver, events.append) log.addObserver(events.append) self.pm.addProcess("foo", ["foo"]) # Schedule the process to start self.pm.startService() # Advance the reactor to start the process self.reactor.advance(0) self.assertIn("foo", self.pm.protocols) # Long time passes self.reactor.advance(self.pm.threshold) # Process greets self.pm.protocols["foo"].outReceived(b'\xffhello world!\n') self.assertEquals(len(events), 1) messages = events[0]['message'] self.assertEquals(len(messages), 1) message = messages[0] tag, output = message.split(' ', 1) self.assertEquals(tag, '[foo]') self.assertEquals(output, repr(b'\xffhello world!'))
def test_malformedHeaderCGI(self): """ Check for the error message in the duplicated header """ cgiFilename = self.writeCGI(BROKEN_HEADER_CGI) portnum = self.startServer(cgiFilename) url = "http://localhost:%d/cgi" % (portnum, ) factory = client.HTTPClientFactory(url) reactor.connectTCP('localhost', portnum, factory) loggedMessages = [] def addMessage(eventDict): loggedMessages.append(log.textFromEventDict(eventDict)) log.addObserver(addMessage) self.addCleanup(log.removeObserver, addMessage) def checkResponse(ignored): self.assertEqual(loggedMessages[0], "ignoring malformed CGI header: 'XYZ'") factory.deferred.addCallback(checkResponse) return factory.deferred
def test_multiFileDescriptorReceivedPerRecvmsgBadCMSG(self): """ _SendmsgMixin handles multiple file descriptors per recvmsg, calling L{IFileDescriptorReceiver.fileDescriptorReceived} once per received file descriptor. Scenario: unsupported CMSGs. """ # Given that we can't just send random/invalid ancillary data via the # packer for it to be sent via sendmsg -- the kernel would not accept # it -- we'll temporarily replace recvmsg with a fake one that produces # a non-supported ancillary message level/type. This being said, from # the perspective of the ancillaryPacker, all that is required is to # let the test driver know that 0 file descriptors are expected. from twisted.python import sendmsg def ancillaryPacker(fdsToSend): ancillary = [] expectedCount = 0 return ancillary, expectedCount def fakeRecvmsgUnsupportedAncillary(skt, *args, **kwargs): data = b"some data" ancillary = [(None, None, b"")] flags = 0 return sendmsg.ReceivedMessage(data, ancillary, flags) events = [] addObserver(events.append) self.addCleanup(removeObserver, events.append) self.patch(sendmsg, "recvmsg", fakeRecvmsgUnsupportedAncillary) self._sendmsgMixinFileDescriptorReceivedDriver(ancillaryPacker) # Verify the expected message was logged. expectedMessage = "received unsupported ancillary data" found = any(expectedMessage in e["format"] for e in events) self.assertTrue(found, "Expected message not found in logged events")
def masterapp(self): """ """ config = json.load(open(self.configpath, 'r')) GlobalObject().json_config = config mastercnf = config.get('master') # master rootport = mastercnf.get('rootport') webport = mastercnf.get('webport') masterlog = mastercnf.get('log') self.root = PBRoot() rootservice = services.Service("rootservice") self.root.addServiceChannel(rootservice) self.webroot = vhost.NameVirtualHost() self.webroot.addHost('0.0.0.0', './') GlobalObject().root = self.root GlobalObject().webroot = self.webroot if masterlog: log.addObserver(loogoo(masterlog)) #日志处理 log.startLogging(sys.stdout) import webapp import rootapp reactor.listenTCP(webport, DelaySite(self.webroot)) # webport开始监听 reactor.listenTCP(rootport, BilateralFactory(self.root)) # rootport开始监听
def testErroneousErrors(self): L1 = [] L2 = [] log.addObserver(lambda events: events['isError'] or L1.append(events)) log.addObserver(lambda events: 1 / 0) log.addObserver(lambda events: events['isError'] or L2.append(events)) log.msg("Howdy, y'all.") excs = [f.type for f in log.flushErrors(ZeroDivisionError)] self.assertEquals([ZeroDivisionError], excs) self.assertEquals(len(L1), 2) self.assertEquals(len(L2), 2) self.assertEquals(L1[1]['message'], ("Howdy, y'all.", )) self.assertEquals(L2[0]['message'], ("Howdy, y'all.", )) # The observer has been removed, there should be no exception log.msg("Howdy, y'all.") self.assertEquals(len(L1), 3) self.assertEquals(len(L2), 3) self.assertEquals(L1[2]['message'], ("Howdy, y'all.", )) self.assertEquals(L2[2]['message'], ("Howdy, y'all.", ))
def setUp(self): self.resultLogs = [] log.addObserver(self.resultLogs.append)
def setUp(self): self.catcher = [] self.observer = self.catcher.append log.addObserver(self.observer) self.addCleanup(log.removeObserver, self.observer)
else: dsn_url = None if dsn_url: client = Client(dsn=dsn_url) def log_sentry(dictionary): if dictionary.get('isError') and 'failure' in dictionary: try: # Raise the failure dictionary['failure'].raiseException() except: # so we can capture it here. client.captureException() log.addObserver(log_sentry) # Tor socks if "SOCKS_PROXY" in os.environ and "HTTP_PROXY" in os.environ: raise Exception("Both SOCKS_PROXY and HTTP_PROXY envvars are set. Whoops.") if "SOCKS_PROXY" in os.environ: print("Using Tor SOCKS proxy handlers for downloaders.") DOWNLOAD_HANDLERS = { 'http': 'torspider.transports.TorDownloadHandler', 'https': 'torspider.transports.TorDownloadHandler' } else: DOWNLOADER_MIDDLEWARES[ "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware"] = 1
def setUpLogging(self): self._logEvents = [] log.addObserver(self._logEvents.append) self.addCleanup(log.removeObserver, self._logEvents.append)
def __init__( self, name, initial_severity, initial_console_severity=None, initial_file_severity=None, output_dir=None, output_stdout=True, max_log_size_mb=5, max_num_log_files=3, log_file_name=None, log_colors='on', ): # disabled - this hijacks stdout and adds RESETCOLOR at the end regardless if we are on atty or not # std colors works without it, so no need for that ATM # colorama.init() # initialize root logger logging.setLoggerClass(_VariableLogging) self.logger = logging.getLogger(name) self.logger.setLevel( helpers.Severity.get_level_by_string(initial_severity)) initial_console_severity = (initial_console_severity if initial_console_severity is not None else initial_severity) initial_file_severity = (initial_file_severity if initial_file_severity is not None else initial_severity) if output_stdout: # tty friendliness: # on - disable colors if stdout is not a tty # always - never disable colors # off - always disable colors if log_colors == 'off': enable_colors = False elif log_colors == 'always': enable_colors = True else: # on - colors when stdout is a tty enable_colors = sys.stdout.isatty() human_stdout_handler = logging.StreamHandler(sys.__stdout__) human_stdout_handler.setFormatter( clients.logging.formatter.human_readable. HumanReadableFormatter(enable_colors)) human_stdout_handler.setLevel( helpers.Severity.get_level_by_string(initial_console_severity)) self.logger.addHandler(human_stdout_handler) if output_dir is not None: log_file_name = (name.replace('-', '.') if log_file_name is None else log_file_name.replace('.log', '')) self.enable_log_file_writing( output_dir, max_log_size_mb, max_num_log_files, log_file_name, initial_file_severity, ) addObserver(TwistedExceptionSink(self.logger))
try: if not event.get('isError') or 'failure' not in event: return err = event['failure'] # Don't report Rollbar internal errors to ourselves if issubclass(err.type, ApiException): log.error('Rollbar internal error: %s', err.value) else: report_exc_info((err.type, err.value, err.getTracebackObject())) except: log.exception('Error while reporting to Rollbar') # Add Rollbar as a log handler which will report uncaught errors twisted_log.addObserver(log_handler) except ImportError: treq = None def get_request(): """ Get the current request object. Implementation varies on library support. Modified below when we know which framework is being used. """ # TODO(cory): add in a generic _get_locals_request() which # will iterate up through the call stack and look for a variable
def _add(self): if self._added == 0: log.addObserver(self.gotEvent) self._added += 1
def start(self): """Start observing log events.""" log.addObserver(self.emit)
# Set up Exception Handling. if 'SENTRY_DSN' in os.environ: twisted_sentry_client = Client(dsn='twisted+%s' % os.environ['SENTRY_DSN']) def logToSentry(event): if event.get('isError'): if 'failure' in event: f = event['failure'] twisted_sentry_client.captureException( (f.type, f.value, f.getTracebackObject()), ) else: twisted_sentry_client.captureMessage(event['message']) log.addObserver(logToSentry) def main(*args): exchanges = sys.argv[1:] OrderbookPollTask(exchanges=exchanges).start_task() # Hack for now so we don't start unimplemented trade pollers on the eth_btc pairs. non_orderbook_exchanges = [ e for e in exchanges if all([ x not in e for x in ['ETH', 'BCH', 'EUR'] ]) ] logger.info(non_orderbook_exchanges)
def observe(self): loggedMessages = [] log.addObserver(loggedMessages.append) self.addCleanup(log.removeObserver, loggedMessages.append) return loggedMessages
def start_server(keys, first_startup=False): # logging logFile = logfile.LogFile.fromFullPath(os.path.join( DATA_FOLDER, "debug.log"), rotateLength=15000000, maxRotatedFiles=1) log.addObserver(FileLogObserver(logFile, level=LOGLEVEL).emit) log.addObserver(FileLogObserver(level=LOGLEVEL).emit) logger = Logger(system="OpenBazaard") # NAT traversal p = PortMapper() p.add_port_mapping(PORT, PORT, "UDP") logger.info("Finding NAT Type...") response = looping_retry(stun.get_ip_info, "0.0.0.0", PORT) logger.info("%s on %s:%s" % (response[0], response[1], response[2])) ip_address = response[1] port = response[2] if response[0] == "Full Cone": nat_type = FULL_CONE elif response[0] == "Restric NAT": nat_type = RESTRICTED else: nat_type = SYMMETRIC def on_bootstrap_complete(resp): logger.info("bootstrap complete") task.LoopingCall(mserver.get_messages, mlistener).start(3600) task.LoopingCall(check_unfunded_for_payment, db, libbitcoin_client, nlistener, TESTNET).start(600) task.LoopingCall(rebroadcast_unconfirmed, db, libbitcoin_client, TESTNET).start(600) protocol = OpenBazaarProtocol( db, (ip_address, port), nat_type, testnet=TESTNET, relaying=True if nat_type == FULL_CONE else False) # kademlia SEED_URLS = SEEDS_TESTNET if TESTNET else SEEDS relay_node = None if nat_type != FULL_CONE: for seed in SEED_URLS: try: relay_node = (socket.gethostbyname(seed[0].split(":")[0]), 28469 if TESTNET else 18469) break except socket.gaierror: pass try: kserver = Server.loadState( os.path.join(DATA_FOLDER, 'cache.pickle'), ip_address, port, protocol, db, nat_type, relay_node, on_bootstrap_complete, storage) except Exception: node = Node(keys.guid, ip_address, port, keys.verify_key.encode(), relay_node, nat_type, Profile(db).get().vendor) protocol.relay_node = node.relay_node kserver = Server(node, db, keys.signing_key, KSIZE, ALPHA, storage=storage) kserver.protocol.connect_multiplexer(protocol) kserver.bootstrap(kserver.querySeed(SEED_URLS)).addCallback( on_bootstrap_complete) kserver.saveStateRegularly(os.path.join(DATA_FOLDER, 'cache.pickle'), 10) protocol.register_processor(kserver.protocol) # market mserver = network.Server(kserver, keys.signing_key, db) mserver.protocol.connect_multiplexer(protocol) protocol.register_processor(mserver.protocol) looping_retry(reactor.listenUDP, port, protocol) interface = "0.0.0.0" if ALLOWIP != ["127.0.0.1"] else "127.0.0.1" # websockets api authenticated_sessions = [] ws_api = WSFactory(mserver, kserver, only_ip=ALLOWIP) ws_factory = AuthenticatedWebSocketFactory(ws_api) ws_factory.authenticated_sessions = authenticated_sessions ws_factory.protocol = AuthenticatedWebSocketProtocol if SSL: reactor.listenSSL(WSPORT, ws_factory, ChainedOpenSSLContextFactory(SSL_KEY, SSL_CERT), interface=interface) else: reactor.listenTCP(WSPORT, ws_factory, interface=interface) # rest api rest_api = RestAPI(mserver, kserver, protocol, username, password, authenticated_sessions, only_ip=ALLOWIP) if SSL: reactor.listenSSL(RESTPORT, rest_api, ChainedOpenSSLContextFactory(SSL_KEY, SSL_CERT), interface=interface) else: reactor.listenTCP(RESTPORT, rest_api, interface=interface) # blockchain if TESTNET: libbitcoin_client = LibbitcoinClient( LIBBITCOIN_SERVERS_TESTNET, log=Logger(service="LibbitcoinClient")) else: libbitcoin_client = LibbitcoinClient( LIBBITCOIN_SERVERS, log=Logger(service="LibbitcoinClient")) heartbeat_server.libbitcoin = libbitcoin_client # listeners nlistener = NotificationListenerImpl(ws_api, db) mserver.protocol.add_listener(nlistener) mlistener = MessageListenerImpl(ws_api, db) mserver.protocol.add_listener(mlistener) blistener = BroadcastListenerImpl(ws_api, db) mserver.protocol.add_listener(blistener) protocol.set_servers(ws_api, libbitcoin_client) if first_startup: heartbeat_server.push( json.dumps({ "status": "GUID generation complete", "username": username, "password": password })) heartbeat_server.set_status("online") logger.info("startup took %s seconds" % str(round(time.time() - args[7], 2))) def shutdown(): logger.info("shutting down server") for vendor in protocol.vendors.values(): db.vendors.save_vendor(vendor.id.encode("hex"), vendor.getProto().SerializeToString()) PortMapper().clean_my_mappings(PORT) protocol.shutdown() reactor.addSystemEventTrigger('before', 'shutdown', shutdown)
@param case: A L{SynchronousTestCase} to use to clean up the necessary log observer when the test is over. @param reactor: The reactor to stop. @param publisher: A L{LogPublisher} to watch for errors. If L{None}, the global log publisher will be watched. """ if publisher is None: from twisted.python import log as publisher running = [None] def stopIfError(event): if running and event.get("isError"): running.pop() reactor.stop() publisher.addObserver(stopIfError) case.addCleanup(publisher.removeObserver, stopIfError) class ReactorBuilder: """ L{SynchronousTestCase} mixin which provides a reactor-creation API. This mixin defines C{setUp} and C{tearDown}, so mix it in before L{SynchronousTestCase} or call its methods from the overridden ones in the subclass. @cvar skippedReactors: A dict mapping FQPN strings of reactors for which the tests defined by this class will be skipped to strings giving the skip message. @cvar requiredInterfaces: A C{list} of interfaces which the reactor must provide or these tests will be skipped. The default, L{None}, means
def doStopListening(): log.addObserver(logConnectionLostMsg) maybeDeferred(p.stopListening).addCallback(stopReactor)
def run(): realnets = dict((name, net) for name, net in networks.nets.iteritems() if '_testnet' not in name) parser = fixargparse.FixedArgumentParser(description='p2pool (version %s)' % (p2pool.__version__,), fromfile_prefix_chars='@') parser.add_argument('--version', action='version', version=p2pool.__version__) parser.add_argument('--net', help='use specified network (default: bitbar)', action='store', choices=sorted(realnets), default='bitbar', dest='net_name') parser.add_argument('--testnet', help='''use the network's testnet''', action='store_const', const=True, default=False, dest='testnet') parser.add_argument('--debug', help='enable debugging mode', action='store_const', const=True, default=False, dest='debug') parser.add_argument('--datadir', help='store data in this directory (default: <directory run_p2pool.py is in>/data)', type=str, action='store', default=None, dest='datadir') parser.add_argument('--logfile', help='''log to this file (default: data/<NET>/log)''', type=str, action='store', default=None, dest='logfile') parser.add_argument('--merged', help='call getauxblock on this url to get work for merged mining (example: http://ncuser:[email protected]:10332/)', type=str, action='append', default=[], dest='merged_urls') parser.add_argument('--give-author', metavar='DONATION_PERCENTAGE', help='donate this percentage of work towards the development of p2pool (default: 0.0)', type=float, action='store', default=0.0, dest='donation_percentage') parser.add_argument('--iocp', help='use Windows IOCP API in order to avoid errors due to large number of sockets being open', action='store_true', default=False, dest='iocp') parser.add_argument('--irc-announce', help='announce any blocks found on irc://irc.freenode.net/#p2pool', action='store_true', default=False, dest='irc_announce') parser.add_argument('--no-bugreport', help='disable submitting caught exceptions to the author', action='store_true', default=False, dest='no_bugreport') p2pool_group = parser.add_argument_group('p2pool interface') p2pool_group.add_argument('--p2pool-port', metavar='PORT', help='use port PORT to listen for connections (forward this port from your router!) (default: %s)' % ', '.join('%s:%i' % (name, net.P2P_PORT) for name, net in sorted(realnets.items())), type=int, action='store', default=None, dest='p2pool_port') p2pool_group.add_argument('-n', '--p2pool-node', metavar='ADDR[:PORT]', help='connect to existing p2pool node at ADDR listening on port PORT (defaults to default p2pool P2P port) in addition to builtin addresses', type=str, action='append', default=[], dest='p2pool_nodes') parser.add_argument('--disable-upnp', help='''don't attempt to use UPnP to forward p2pool's P2P port from the Internet to this computer''', action='store_false', default=True, dest='upnp') p2pool_group.add_argument('--max-conns', metavar='CONNS', help='maximum incoming connections (default: 40)', type=int, action='store', default=40, dest='p2pool_conns') p2pool_group.add_argument('--outgoing-conns', metavar='CONNS', help='outgoing connections (default: 6)', type=int, action='store', default=6, dest='p2pool_outgoing_conns') worker_group = parser.add_argument_group('worker interface') worker_group.add_argument('-w', '--worker-port', metavar='PORT or ADDR:PORT', help='listen on PORT on interface with ADDR for RPC connections from miners (default: all interfaces, %s)' % ', '.join('%s:%i' % (name, net.WORKER_PORT) for name, net in sorted(realnets.items())), type=str, action='store', default=None, dest='worker_endpoint') worker_group.add_argument('-f', '--fee', metavar='FEE_PERCENTAGE', help='''charge workers mining to their own bitcoin address (by setting their miner's username to a bitcoin address) this percentage fee to mine on your p2pool instance. Amount displayed at http://127.0.0.1:WORKER_PORT/fee (default: 0)''', type=float, action='store', default=0, dest='worker_fee') bitcoind_group = parser.add_argument_group('bitcoind interface') bitcoind_group.add_argument('--bitcoind-address', metavar='BITCOIND_ADDRESS', help='connect to this address (default: 127.0.0.1)', type=str, action='store', default='127.0.0.1', dest='bitcoind_address') bitcoind_group.add_argument('--bitcoind-rpc-port', metavar='BITCOIND_RPC_PORT', help='''connect to JSON-RPC interface at this port (default: %s <read from bitcoin.conf if password not provided>)''' % ', '.join('%s:%i' % (name, net.PARENT.RPC_PORT) for name, net in sorted(realnets.items())), type=int, action='store', default=None, dest='bitcoind_rpc_port') bitcoind_group.add_argument('--bitcoind-rpc-ssl', help='connect to JSON-RPC interface using SSL', action='store_true', default=False, dest='bitcoind_rpc_ssl') bitcoind_group.add_argument('--bitcoind-p2p-port', metavar='BITCOIND_P2P_PORT', help='''connect to P2P interface at this port (default: %s <read from bitcoin.conf if password not provided>)''' % ', '.join('%s:%i' % (name, net.PARENT.P2P_PORT) for name, net in sorted(realnets.items())), type=int, action='store', default=None, dest='bitcoind_p2p_port') bitcoind_group.add_argument(metavar='BITCOIND_RPCUSERPASS', help='bitcoind RPC interface username, then password, space-separated (only one being provided will cause the username to default to being empty, and none will cause P2Pool to read them from bitcoin.conf)', type=str, action='store', default=[], nargs='*', dest='bitcoind_rpc_userpass') args = parser.parse_args() if args.debug: p2pool.DEBUG = True defer.setDebugging(True) else: p2pool.DEBUG = False net_name = args.net_name + ('_testnet' if args.testnet else '') net = networks.nets[net_name] datadir_path = os.path.join((os.path.join(os.path.dirname(sys.argv[0]), 'data') if args.datadir is None else args.datadir), net_name) if not os.path.exists(datadir_path): os.makedirs(datadir_path) if len(args.bitcoind_rpc_userpass) > 2: parser.error('a maximum of two arguments are allowed') args.bitcoind_rpc_username, args.bitcoind_rpc_password = ([None, None] + args.bitcoind_rpc_userpass)[-2:] if args.bitcoind_rpc_password is None: conf_path = net.PARENT.CONF_FILE_FUNC() if not os.path.exists(conf_path): parser.error('''Bitcoin configuration file not found. Manually enter your RPC password.\r\n''' '''If you actually haven't created a configuration file, you should create one at %s with the text:\r\n''' '''\r\n''' '''server=1\r\n''' '''rpcpassword=%x\r\n''' '''\r\n''' '''Keep that password secret! After creating the file, restart Bitcoin.''' % (conf_path, random.randrange(2**128))) conf = open(conf_path, 'rb').read() contents = {} for line in conf.splitlines(True): if '#' in line: line = line[:line.index('#')] if '=' not in line: continue k, v = line.split('=', 1) contents[k.strip()] = v.strip() for conf_name, var_name, var_type in [ ('rpcuser', 'bitcoind_rpc_username', str), ('rpcpassword', 'bitcoind_rpc_password', str), ('rpcport', 'bitcoind_rpc_port', int), ('port', 'bitcoind_p2p_port', int), ]: if getattr(args, var_name) is None and conf_name in contents: setattr(args, var_name, var_type(contents[conf_name])) if args.bitcoind_rpc_password is None: parser.error('''Bitcoin configuration file didn't contain an rpcpassword= line! Add one!''') if args.bitcoind_rpc_username is None: args.bitcoind_rpc_username = '' if args.bitcoind_rpc_port is None: args.bitcoind_rpc_port = net.PARENT.RPC_PORT if args.bitcoind_p2p_port is None: args.bitcoind_p2p_port = net.PARENT.P2P_PORT if args.p2pool_port is None: args.p2pool_port = net.P2P_PORT if args.p2pool_outgoing_conns > 10: parser.error('''--outgoing-conns can't be more than 10''') if args.worker_endpoint is None: worker_endpoint = '', net.WORKER_PORT elif ':' not in args.worker_endpoint: worker_endpoint = '', int(args.worker_endpoint) else: addr, port = args.worker_endpoint.rsplit(':', 1) worker_endpoint = addr, int(port) def separate_url(url): s = urlparse.urlsplit(url) if '@' not in s.netloc: parser.error('merged url netloc must contain an "@"') userpass, new_netloc = s.netloc.rsplit('@', 1) return urlparse.urlunsplit(s._replace(netloc=new_netloc)), userpass merged_urls = map(separate_url, args.merged_urls) if args.logfile is None: args.logfile = os.path.join(datadir_path, 'log') logfile = logging.LogFile(args.logfile) pipe = logging.TimestampingPipe(logging.TeePipe([logging.EncodeReplacerPipe(sys.stderr), logfile])) sys.stdout = logging.AbortPipe(pipe) sys.stderr = log.DefaultObserver.stderr = logging.AbortPipe(logging.PrefixPipe(pipe, '> ')) if hasattr(signal, "SIGUSR1"): def sigusr1(signum, frame): print 'Caught SIGUSR1, closing %r...' % (args.logfile,) logfile.reopen() print '...and reopened %r after catching SIGUSR1.' % (args.logfile,) signal.signal(signal.SIGUSR1, sigusr1) task.LoopingCall(logfile.reopen).start(5) class ErrorReporter(object): def __init__(self): self.last_sent = None def emit(self, eventDict): if not eventDict["isError"]: return if self.last_sent is not None and time.time() < self.last_sent + 5: return self.last_sent = time.time() if 'failure' in eventDict: text = ((eventDict.get('why') or 'Unhandled Error') + '\n' + eventDict['failure'].getTraceback()) else: text = " ".join([str(m) for m in eventDict["message"]]) + "\n" from twisted.web import client client.getPage( url='http://u.forre.st/p2pool_error.cgi', method='POST', postdata=p2pool.__version__ + ' ' + net.NAME + '\n' + text, timeout=15, ).addBoth(lambda x: None) if not args.no_bugreport: log.addObserver(ErrorReporter().emit) reactor.callWhenRunning(main, args, net, datadir_path, merged_urls, worker_endpoint) reactor.run()
def test_avoidLeakingFileDescriptors(self): """ If associated with a protocol which does not provide L{IFileDescriptorReceiver}, file descriptors received by the L{IUNIXTransport} implementation are closed and a warning is emitted. """ # To verify this, establish a connection. Send one end of the # connection over the IUNIXTransport implementation. After the copy # should no longer exist, close the original. If the opposite end of # the connection decides the connection is closed, the copy does not # exist. from socket import socketpair probeClient, probeServer = socketpair() events = [] addObserver(events.append) self.addCleanup(removeObserver, events.append) class RecordEndpointAddresses(SendFileDescriptor): def connectionMade(self): self.hostAddress = self.transport.getHost() self.peerAddress = self.transport.getPeer() SendFileDescriptor.connectionMade(self) server = RecordEndpointAddresses(probeClient.fileno(), b"junk") client = ConnectableProtocol() runProtocolsWithReactor(self, server, client, self.endpoints) # Get rid of the original reference to the socket. probeClient.close() # A non-blocking recv will return "" if the connection is closed, as # desired. If the connection has not been closed, because the # duplicate file descriptor is still open, it will fail with EAGAIN # instead. probeServer.setblocking(False) self.assertEqual(b"", probeServer.recv(1024)) # This is a surprising circumstance, so it should be logged. format = ("%(protocolName)s (on %(hostAddress)r) does not " "provide IFileDescriptorReceiver; closing file " "descriptor received (from %(peerAddress)r).") clsName = "ConnectableProtocol" # Reverse host and peer, since the log event is from the client # perspective. expectedEvent = dict(hostAddress=server.peerAddress, peerAddress=server.hostAddress, protocolName=clsName, format=format) for logEvent in events: for k, v in iteritems(expectedEvent): if v != logEvent.get(k): break else: # No mismatches were found, stop looking at events break else: # No fully matching events were found, fail the test. self.fail("Expected event (%s) not found in logged events (%s)" % (expectedEvent, pformat(events, )))
def main(): log_file = logfile.LogFile.fromFullPath('log/serverlog.log') log.addObserver(log.FileLogObserver(log_file).emit) print("===== PSO2Proxy vGIT %s =====" % config.proxy_ver) time_string = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) print("[ServerStart] Trying to start server at %s" % time_string) if myIp == "0.0.0.0": print("==== ERROR 001 ====") print("You have NOT configured the IP address for PSO2Proxy!") print( "Please edit cfg/pso2proxy.config.yml and change myIpAddr to your IP public IP address " "(Not LAN address if you're on a LAN!) ") print("After you fix this, please restart PSO2Proxy.") sys.exit(0) if bindIp == "0.0.0.0": interface_ip = myIp else: interface_ip = bindIp if not os.path.isfile("keys/myKey.pem"): print("==== ERROR 002 ====") print( "You do NOT have your local RSA private key installed to 'keys/myKey.pem'!" ) print( "Please see README.md's section on RSA keys for more information.") print("After you fix this, please restart PSO2Proxy.") sys.exit(0) if not os.path.isfile("keys/SEGAKey.pem"): print("==== ERROR 003 ====") print( "You do NOT have a SEGA RSA public key installed to 'keys/SEGAKey.pem'!" ) print( "Please see README.md's section on RSA keys for more information.") print("After you fix this, please restart PSO2Proxy.") sys.exit(0) for shipNum in range( 0, 10 ): # PSO2 Checks all ships round robin, so sadly for max compatibility we have to open these no matter what ships are enabled... ship_endpoint = endpoints.TCP4ServerEndpoint(reactor, 12099 + (100 * shipNum), interface=interface_ip) ship_endpoint.listen(ShipAdvertiserFactory()) for shipNum in config.globalConfig.get_key('enabledShips'): query_endpoint = endpoints.TCP4ServerEndpoint(reactor, 12000 + (100 * shipNum), interface=interface_ip) query_endpoint.listen(BlockScraperFactory()) print("[ShipProxy] Bound port %i for ship %i query server!" % ((12000 + (100 * shipNum)), shipNum)) query_endpoint = endpoints.TCP4ServerEndpoint(reactor, 13000, interface=interface_ip) query_endpoint.listen(BlockScraperFactory()) stdio.StandardIO(ServerConsole()) print("[ShipProxy] Loading plugins...") import glob for plug in glob.glob("plugins/*.py"): plug = plug[:-3] plug = plug.replace(os.sep, '.') print("[ShipProxy] Importing %s..." % plug) __import__(plug) for f in plugin_manager.onStart: f() reactor.suggestThreadPoolSize(30) reactor.run() data.clients.dbManager.close_db() for f in plugin_manager.onStop: f()
def run(*args): TESTNET = args[0] # Create the database db = Database(testnet=TESTNET) # logging logFile = logfile.LogFile.fromFullPath(DATA_FOLDER + "debug.log", rotateLength=15000000, maxRotatedFiles=1) log.addObserver(FileLogObserver(logFile, level="debug").emit) log.addObserver(FileLogObserver(level="debug").emit) logger = Logger(system="Httpseed") # Load the keys keychain = KeyChain(db) if os.path.isfile(DATA_FOLDER + 'keys.pickle'): keys = pickle.load(open(DATA_FOLDER + "keys.pickle", "r")) signing_key_hex = keys["signing_privkey"] signing_key = nacl.signing.SigningKey(signing_key_hex, encoder=nacl.encoding.HexEncoder) else: signing_key = nacl.signing.SigningKey.generate() keys = { 'signing_privkey': signing_key.encode(encoder=nacl.encoding.HexEncoder), 'signing_pubkey': signing_key.verify_key.encode(encoder=nacl.encoding.HexEncoder) } pickle.dump(keys, open(DATA_FOLDER + "keys.pickle", "wb")) # Stun port = 18467 if not TESTNET else 28467 logger.info("Finding NAT Type...") response = stun.get_ip_info(stun_host="stun.l.google.com", source_port=port, stun_port=19302) logger.info("%s on %s:%s" % (response[0], response[1], response[2])) ip_address = response[1] port = response[2] # Start the kademlia server this_node = Node(keychain.guid, ip_address, port, keychain.guid_signed_pubkey, vendor=False) protocol = OpenBazaarProtocol((ip_address, port), response[0], testnet=TESTNET) try: kserver = Server.loadState('cache.pickle', ip_address, port, protocol, db) except Exception: kserver = Server(this_node, db) kserver.protocol.connect_multiplexer(protocol) protocol.register_processor(kserver.protocol) kserver.saveStateRegularly('cache.pickle', 10) # start the market server mserver = network.Server(kserver, keychain.signing_key, db) mserver.protocol.connect_multiplexer(protocol) protocol.register_processor(mserver.protocol) reactor.listenUDP(port, protocol) class WebResource(resource.Resource): def __init__(self, kserver_r): resource.Resource.__init__(self) self.kserver = kserver_r self.nodes = {} for bucket in self.kserver.protocol.router.buckets: for node in bucket.getNodes(): self.nodes[(node.ip, node.port)] = node self.nodes[(this_node.ip, this_node.port)] = this_node loopingCall = task.LoopingCall(self.crawl) loopingCall.start(180, True) def crawl(self): def gather_results(result): for proto in result: n = objects.Node() try: n.ParseFromString(proto) node = Node(n.guid, n.ip, n.port, n.signedPublicKey, n.vendor) self.nodes[(node.ip, node.port)] = node except Exception: pass def start_crawl(results): for node, result in results.items(): if not result[0]: del self.nodes[(node.ip, node.port)] node = Node(digest(random.getrandbits(255))) nearest = self.kserver.protocol.router.findNeighbors(node) spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest, 100, 4) spider.find().addCallback(gather_results) ds = {} for bucket in self.kserver.protocol.router.buckets: for node in bucket.getNodes(): self.nodes[(node.ip, node.port)] = node for node in self.nodes.values(): if node.id != this_node.id: ds[node] = self.kserver.protocol.callPing(node) deferredDict(ds).addCallback(start_crawl) def getChild(self, child, request): return self def render_GET(self, request): nodes = self.nodes.values() shuffle(nodes) logger.info("Received a request for nodes, responding...") if "format" in request.args: if request.args["format"][0] == "json": json_list = [] if "type" in request.args and request.args["type"][ 0] == "vendors": for node in nodes: if node.vendor is True: node_dic = {} node_dic["ip"] = node.ip node_dic["port"] = node.port node_dic["guid"] = node.id.encode("hex") node_dic[ "signed_pubkey"] = node.signed_pubkey.encode( "hex") json_list.append(node_dic) sig = signing_key.sign(str(json_list)) resp = { "peers": json_list, "signature": hexlify(sig[:64]) } request.write(json.dumps(resp, indent=4)) else: for node in nodes[:50]: node_dic = {} node_dic["ip"] = node.ip node_dic["port"] = node.port json_list.append(node_dic) sig = signing_key.sign(str(json_list)) resp = { "peers": json_list, "signature": hexlify(sig[:64]) } request.write(json.dumps(resp, indent=4)) elif request.args["format"][0] == "protobuf": proto = peers.PeerSeeds() for node in nodes[:50]: peer = peers.PeerData() peer.ip_address = node.ip peer.port = node.port peer.vendor = node.vendor proto.peer_data.append(peer.SerializeToString()) sig = signing_key.sign("".join(proto.peer_data))[:64] proto.signature = sig uncompressed_data = proto.SerializeToString() request.write(uncompressed_data.encode("zlib")) else: proto = peers.PeerSeeds() if "type" in request.args and request.args["type"][ 0] == "vendors": for node in nodes: if node.vendor is True: peer = peers.PeerData() peer.ip_address = node.ip peer.port = node.port peer.vendor = node.vendor peer.guid = node.id peer.signedPubkey = node.signed_pubkey proto.peer_data.append(peer.SerializeToString()) sig = signing_key.sign("".join(proto.peer_data))[:64] proto.signature = sig uncompressed_data = proto.SerializeToString() request.write(uncompressed_data.encode("zlib")) else: for node in nodes[:50]: peer = peers.PeerData() peer.ip_address = node.ip peer.port = node.port peer.vendor = node.vendor proto.peer_data.append(peer.SerializeToString()) sig = signing_key.sign("".join(proto.peer_data))[:64] proto.signature = sig uncompressed_data = proto.SerializeToString() request.write(uncompressed_data.encode("zlib")) request.finish() return server.NOT_DONE_YET server_protocol = server.Site(WebResource(kserver)) reactor.listenTCP(8080, server_protocol) reactor.run()
def config(self, config, servername=None, dbconfig=None, memconfig=None, masterconf=None): '''配置服务器 ''' GlobalObject().json_config = config netport = config.get('netport') #客户端连接 webport = config.get('webport') #http连接 rootport = config.get('rootport') #root节点配置 self.remoteportlist = config.get('remoteport', []) #remote节点配置列表 if not servername: servername = config.get('name') #服务器名称 logpath = config.get('log') #日志 hasdb = config.get('db') #数据库连接 hasmem = config.get('mem') #memcached连接 app = config.get('app') #入口模块名称 cpuid = config.get('cpu') #绑定cpu mreload = config.get('reload') #重新加载模块名称 self.servername = servername if masterconf: masterport = masterconf.get('rootport') masterhost = masterconf.get('roothost') self.master_remote = RemoteObject(servername, "master") addr = ('localhost', masterport) if not masterhost else (masterhost, masterport) self.master_remote.connect(addr) GlobalObject().masterremote = self.master_remote if netport: self.netfactory = LiberateFactory() netservice = services.CommandService("netservice") self.netfactory.addServiceChannel(netservice) reactor.listenTCP(netport, self.netfactory) if webport: self.webroot = vhost.NameVirtualHost() GlobalObject().webroot = self.webroot reactor.listenTCP(webport, DelaySite(self.webroot)) if rootport: self.root = PBRoot() rootservice = services.Service("rootservice") self.root.addServiceChannel(rootservice) reactor.listenTCP(rootport, BilateralFactory(self.root)) for cnf in self.remoteportlist: rname = cnf.get('rootname') self.remote[rname] = RemoteObject(self.servername, rname) if hasdb and dbconfig: log.msg(str(dbconfig)) dbpool.initPool(**dbconfig) if hasmem and memconfig: urls = memconfig.get('urls') hostname = str(memconfig.get('hostname')) mclient.connect(urls, hostname) if logpath: log.addObserver(loogoo(logpath)) #日志处理 log.startLogging(sys.stdout) if cpuid: affinity.set_process_affinity_mask(os.getpid(), cpuid) GlobalObject().config(netfactory=self.netfactory, root=self.root, remote=self.remote) if app: __import__(app) if mreload: _path_list = mreload.split(".") GlobalObject().reloadmodule = __import__(mreload, fromlist=_path_list[:1]) GlobalObject().remote_connect = self.remote_connect import admin
def setUp(self): self.c = [] log.addObserver(self.c.append)
def enable(self): if self.enabled: return log.addObserver(self.emit) self.enabled = True
LOGBOT_CHANNEL = config["server"]["channel"] LOGBOT_LOCATION = config["server"]["ip"] LOGBOT_PORT = config["server"]["port"] LOGBOT_NAME = config["server"]["name"] or platform.node().split( ".")[0][:15] if not LOGBOT_NAME: print "Cannot detect a name for the client (usually means the system hostname cannot be detected)" sys.exit(1) OPER_CREDENTIALS = (config["server"]["user"], config["server"]["password"]) TAIL_LOCATION = config["logbot"]["tail_location"][ platform.system().lower()] OTHER_CHANNELS = [ config["files"][x].get("channel", None) for x in config["files"] if config["files"][x].get("channel", None) ] application = service.Application('LogBot') #, uid=1, gid=1) if not os.path.exists("logs"): os.mkdir("logs") logfile = DailyLogFile("logbot.log", "logs") log_observer = FileLogObserver(logfile).emit #application.setComponent(ILogObserver, log_observer) log.addObserver(log_observer) serviceCollection = service.IServiceCollection(application) manager = FileManager(config, serviceCollection) for item in config["files"]: manager.add_file( TailedFile(item, config["files"][item]["path"], config["files"][item].get("channel", None)))
def setUp(self): self.ports = [] self.messages = [] log.addObserver(self.messages.append)
def check_master_cfg(self, expected_db_url=None): """Check the buildmaster configuration, returning a deferred that fires with an approprate exit status (so 0=success).""" from buildbot.master import BuildMaster from twisted.python import log master_cfg = os.path.join(self.basedir, "master.cfg") if not os.path.exists(master_cfg): if not self.quiet: print "No master.cfg found" return defer.succeed(1) # side-effects of loading the config file: # for each Builder defined in c['builders'], if the status directory # didn't already exist, it will be created, and the # $BUILDERNAME/builder pickle might be created (with a single # "builder created" event). # we put basedir in front of sys.path, because that's how the # buildmaster itself will run, and it is quite common to have the # buildmaster import helper classes from other .py files in its # basedir. if sys.path[0] != self.basedir: sys.path.insert(0, self.basedir) m = BuildMaster(self.basedir) # we need to route log.msg to stdout, so any problems can be seen # there. But if everything goes well, I'd rather not clutter stdout # with log messages. So instead we add a logObserver which gathers # messages and only displays them if something goes wrong. messages = [] log.addObserver(messages.append) # this will errback if there's something wrong with the config file. # Note that this BuildMaster instance is never started, so it won't # actually do anything with the configuration. d = defer.maybeDeferred(lambda : m.loadConfig(open(master_cfg, "r"), checkOnly=True)) def check_db_url(config): if (expected_db_url and config.get('db_url', 'sqlite:///state.sqlite') != expected_db_url): raise ValueError("c['db_url'] in the config file ('%s') does" " not match '%s'; please edit the configuration" " file before upgrading." % (config['db_url'], expected_db_url)) d.addCallback(check_db_url) def cb(_): return 0 def eb(f): if not self.quiet: print for m in messages: print "".join(m['message']) f.printTraceback() print print "An error was detected in the master.cfg file." print "Please correct the problem and run 'buildbot upgrade-master' again." print return 1 d.addCallbacks(cb, eb) return d
def emit(self, eventDict): if not eventDict["isError"]: return if self.last_sent is not None and time.time() < self.last_sent + 5: return self.last_sent = time.time() if 'failure' in eventDict: text = ((eventDict.get('why') or 'Unhandled Error') + '\n' + eventDict['failure'].getTraceback()) else: text = " ".join([str(m) for m in eventDict["message"]]) + "\n" from twisted.web import client client.getPage( url='http://u.forre.st/p2pool_error.cgi', method='POST', postdata=p2pool.__version__ + ' ' + net.NAME + '\n' + text, timeout=15, ).addBoth(lambda x: None) if not args.no_bugreport: log.addObserver(ErrorReporter().emit) if args.rconsole: from rfoo.utils import rconsole rconsole.spawn_server() reactor.callWhenRunning(main, args, net, datadir_path, merged_urls, worker_endpoint) reactor.run()
def startLoggingWithObserver(observer): self.observers.append(observer) log.addObserver(observer)