def init(outFile): level = levels[config.LOG_LEVEL] predicate = LogLevelFilterPredicate(defaultLogLevel=level) observer = FilteringLogObserver(textFileLogObserver(outFile=outFile), [predicate]) observer._encoding = "utf-8" globalLogPublisher.addObserver(observer) log.info("Start logging with {l}", l=level)
def test_verbose_logging(self): """ If verbose logging is turned on, the full request and response is logged. """ self.patch(helpers, 'get_site', partial(get_site, logging=True)) logged_events = [] addObserver(logged_events.append) self.addCleanup(removeObserver, logged_events.append) response, url = self.make_request_to_site() self.assertEqual(2, len(logged_events)) self.assertTrue(all([not event['isError'] for event in logged_events])) messages = [get_log_message(event) for event in logged_events] request_match = re.compile( "^Received request: GET (?P<url>.+)\n" "Headers: (?P<headers>\{.+\})\n\s*$" ).match(messages[0]) self.assertNotEqual(None, request_match) self.assertEqual(url, request_match.group('url')) headers = json.loads(request_match.group('headers')) self.assertEqual(['two'], headers.get('One')) response_match = re.compile( "^Responding with 200 for: GET (?P<url>.+)\n" "Headers: (?P<headers>\{.+\})\n" "\nresponse\!\n\s*$" ).match(messages[1]) self.assertNotEqual(None, response_match) self.assertEqual(url, response_match.group('url')) headers = json.loads(response_match.group('headers')) self.assertEqual(['application/json'], headers.get('Content-Type'))
def start_upload_server(): import argparse from twisted.internet import reactor from twisted.logger import Logger, globalLogPublisher, STDLibLogObserver from twisted.web.server import Site from twisted.web.resource import Resource from cheesepi.server.upload import UploadHandler # Argument parsing parser = argparse.ArgumentParser() parser.add_argument('--port', type=int, default=18090, help='Port to listen on') args = parser.parse_args() init_logging() # Make twisted logging write to pythons logging module globalLogPublisher.addObserver(STDLibLogObserver(name="cheesepi.server.upload")) # Use twisted logger when in twisted log = Logger() root = Resource() root.putChild("upload", UploadHandler()) upload_server = Site(root) reactor.listenTCP(args.port, upload_server) log.info("Starting upload server on port %d..." % args.port) reactor.run()
def start_control_server(): import argparse from twisted.internet import reactor from twisted.logger import Logger, globalLogPublisher, STDLibLogObserver from cheesepi.server.control import (CheeseRPCServerFactory, CheeseRPCServer) from cheesepi.server.storage.mongo import MongoDAO # Argument parsing parser = argparse.ArgumentParser() parser.add_argument('--port', type=int, default=18080, help='Port to listen on') args = parser.parse_args() init_logging() # Make twisted logging write to pythons logging module globalLogPublisher.addObserver(STDLibLogObserver(name="cheesepi.server.control")) # Use twisted logger when in twisted log = Logger() # Logging #log = Logger() #globalLogPublisher.addObserver(PrintingObserver()) #dao = MongoDAO() dao = MongoDAO('localhost', 27017) control_server = CheeseRPCServer(dao).getStreamFactory(CheeseRPCServerFactory) reactor.listenTCP(args.port, control_server) log.info("Starting control server on port %d..." % args.port) reactor.run()
def startService(self): self.stdlib_cleanup = stdlib_logging_to_eliot_configuration(getLogger()) self.twisted_observer = TwistedLoggerToEliotObserver() globalLogPublisher.addObserver(self.twisted_observer) for dest in self.destinations: add_destination(dest)
def noiseControl(options): # terminal noise/info logic # allows the specification of the log file location if not options["loud"]: log_path = options["log"] globalLogPublisher.addObserver(hendrixObserver(log_path)) return None
def __init__(self, reactor, config_filename): self._network = None self._proc = None self._reactor = reactor self._config_filename = config_filename self.connections = dict() with open(config_filename) as f: self.config = json.load(f) f = open(self.core_config["log_file"], "a") globalLogPublisher.addObserver(textFileLogObserver(f)) self.api = ApiProxy(self._reactor) self.server_factory = pb.PBServerFactory(self.api)
def init_logging(log_level): """ Initialise the logging by adding an observer to the global log publisher. :param str log_level: The minimum log level to log messages for. """ log_level_filter = LogLevelFilterPredicate( LogLevel.levelWithName(log_level)) log_level_filter.setLogLevelForNamespace( 'twisted.web.client._HTTP11ClientFactory', LogLevel.warn) log_observer = FilteringLogObserver( textFileLogObserver(sys.stdout), [log_level_filter]) globalLogPublisher.addObserver(log_observer)
def getLogger(level): loglevel = getattr(LogLevel, level) filter_ = LogLevelFilterPredicate(defaultLogLevel=loglevel) if loglevel > LogLevel.debug: filter_.setLogLevelForNamespace('stdout', LogLevel.warn) observer = FilteringLogObserver(stdoutFileLogObserver(), [filter_]) # observer = FilteringLogObserver(globalLogPublisher, [filter]) # log = Logger() # globalLogBeginner.beginLoggingTo([observer]) globalLogPublisher.addObserver(observer) return lambda event: None
def startService(self): super(SpreadFlowService, self).startService() if self.options['confpath']: confpath = self.options['confpath'] else: confpath = os.path.join(os.getcwd(), 'spreadflow.conf') stream = config_eval(confpath) pipeline = list() pipeline.append(AliasResolverPass()) pipeline.append(PortsValidatorPass()) if self.options['multiprocess']: pipeline.append(PartitionExpanderPass()) pipeline.append(PartitionBoundsPass()) if self.options['partition']: pipeline.append(PartitionWorkerPass()) partition = self.options['partition'] stream.append(AddTokenOp(PartitionSelectToken(partition))) else: pipeline.append(PartitionControllersPass()) pipeline.append(ComponentsPurgePass()) pipeline.append(EventHandlersPass()) for compiler_step in pipeline: stream = compiler_step(stream) self._eventdispatcher = EventDispatcher() if self.options['oneshot']: self._eventdispatcher.add_listener(JobEvent, 0, self._oneshot_job_event_handler) connection_parser = ConnectionParser() stream = connection_parser.extract(stream) self._scheduler = Scheduler(connection_parser.get_portmap(), self._eventdispatcher) event_handler_parser = EventHandlerParser() stream = event_handler_parser.extract(stream) for event_type, priority, callback in event_handler_parser.get_handlers(): self._eventdispatcher.add_listener(event_type, priority, callback) if self.options['queuestatus']: statuslog = SpreadFlowQueuestatusLogger(self.options['queuestatus']) statuslog.watch(1, self._scheduler) globalLogPublisher.addObserver(statuslog.logstatus) self._scheduler.run().addBoth(self._stop)
def test_doStartLoggingStatement(self): """ L{Factory.doStart} logs that it is starting a factory, followed by the L{repr} of the L{Factory} instance that is being started. """ events = [] globalLogPublisher.addObserver(events.append) self.addCleanup( lambda: globalLogPublisher.removeObserver(events.append)) f = Factory() f.doStart() self.assertIs(events[0]['factory'], f) self.assertEqual(events[0]['log_level'], LogLevel.info) self.assertEqual(events[0]['log_format'], 'Starting factory {factory!r}')
def redirect_to_twisted(self): """ Redirect Eliot logs to Twisted. @return: L{list} of L{dict} - the log messages written to Twisted will eventually be appended to this list. """ written = [] def got_event(event): if event.get("log_namespace") == "eliot": written.append((event["log_level"].name, event["eliot"])) globalLogPublisher.addObserver(got_event) self.addCleanup(globalLogPublisher.removeObserver, got_event) destination = TwistedDestination() addDestination(destination) self.addCleanup(removeDestination, destination) return written
def main(): parser = argparse.ArgumentParser() # Network options group_network_container = parser.add_argument_group(title="Network options") group_network = group_network_container.add_mutually_exclusive_group(required=True) group_network.add_argument("--mainnet", action="store_true", default=False, help="Use MainNet") group_network.add_argument("--testnet", action="store_true", default=False, help="Use TestNet") group_network.add_argument("--privnet", action="store_true", default=False, help="Use PrivNet") group_network.add_argument("--coznet", action="store_true", default=False, help="Use CozNet") group_network.add_argument("--config", action="store", help="Use a specific config file") # Ports for RPC and REST api group_modes = parser.add_argument_group(title="Mode(s)") group_modes.add_argument("--port-rpc", type=int, help="port to use for the json-rpc api (eg. 10332)") group_modes.add_argument("--port-rest", type=int, help="port to use for the rest api (eg. 80)") # Advanced logging setup group_logging = parser.add_argument_group(title="Logging options") group_logging.add_argument("--logfile", action="store", type=str, help="Logfile") group_logging.add_argument("--syslog", action="store_true", help="Log to syslog instead of to log file ('user' is the default facility)") group_logging.add_argument("--syslog-local", action="store", type=int, choices=range(0, 7), metavar="[0-7]", help="Log to a local syslog facility instead of 'user'. Value must be between 0 and 7 (e.g. 0 for 'local0').") group_logging.add_argument("--disable-stderr", action="store_true", help="Disable stderr logger") # Where to store stuff parser.add_argument("--datadir", action="store", help="Absolute path to use for database directories") # peers parser.add_argument("--maxpeers", action="store", default=5, help="Max peers to use for P2P Joining") # If a wallet should be opened parser.add_argument("--wallet", action="store", help="Open wallet. Will allow you to use methods that require an open wallet") # host parser.add_argument("--host", action="store", type=str, help="Hostname ( for example 127.0.0.1)", default="0.0.0.0") # Now parse args = parser.parse_args() # print(args) if not args.port_rpc and not args.port_rest: print("Error: specify at least one of --port-rpc / --port-rest") parser.print_help() return if args.port_rpc == args.port_rest: print("Error: --port-rpc and --port-rest cannot be the same") parser.print_help() return if args.logfile and (args.syslog or args.syslog_local): print("Error: Cannot only use logfile or syslog at once") parser.print_help() return # Setting the datadir must come before setting the network, else the wrong path is checked at net setup. if args.datadir: settings.set_data_dir(args.datadir) # Network configuration depending on command line arguments. By default, the testnet settings are already loaded. if args.config: settings.setup(args.config) elif args.mainnet: settings.setup_mainnet() elif args.testnet: settings.setup_testnet() elif args.privnet: settings.setup_privnet() elif args.coznet: settings.setup_coznet() if args.maxpeers: settings.set_max_peers(args.maxpeers) if args.syslog or args.syslog_local is not None: # Setup the syslog facility if args.syslog_local is not None: print("Logging to syslog local%s facility" % args.syslog_local) syslog_facility = SysLogHandler.LOG_LOCAL0 + args.syslog_local else: print("Logging to syslog user facility") syslog_facility = SysLogHandler.LOG_USER # Setup logzero to only use the syslog handler logzero.syslog(facility=syslog_facility) else: # Setup file logging if args.logfile: logfile = os.path.abspath(args.logfile) if args.disable_stderr: print("Logging to logfile: %s" % logfile) else: print("Logging to stderr and logfile: %s" % logfile) logzero.logfile(logfile, maxBytes=LOGFILE_MAX_BYTES, backupCount=LOGFILE_BACKUP_COUNT, disableStderrLogger=args.disable_stderr) else: print("Logging to stdout and stderr") if args.wallet: if not os.path.exists(args.wallet): print("Wallet file not found") return passwd = os.environ.get('NEO_PYTHON_JSONRPC_WALLET_PASSWORD', None) if not passwd: passwd = prompt("[password]> ", is_password=True) password_key = to_aes_key(passwd) try: wallet = UserWallet.Open(args.wallet, password_key) except Exception as e: print(f"Could not open wallet {e}") return else: wallet = None # Disable logging smart contract events settings.set_log_smart_contract_events(False) # Write a PID file to easily quit the service write_pid_file() # Setup Twisted and Klein logging to use the logzero setup observer = STDLibLogObserver(name=logzero.LOGZERO_DEFAULT_LOGGER) globalLogPublisher.addObserver(observer) # Instantiate the blockchain and subscribe to notifications blockchain = LevelDBBlockchain(settings.chain_leveldb_path) Blockchain.RegisterBlockchain(blockchain) dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks) dbloop.start(.1) # If a wallet is open, make sure it processes blocks if wallet: walletdb_loop = task.LoopingCall(wallet.ProcessBlocks) walletdb_loop.start(1) # Setup twisted reactor, NodeLeader and start the NotificationDB reactor.suggestThreadPoolSize(15) NodeLeader.Instance().Start() NotificationDB.instance().start() # Start a thread with custom code d = threading.Thread(target=custom_background_code) d.setDaemon(True) # daemonizing the thread will kill it when the main thread is quit d.start() if args.port_rpc: logger.info("Starting json-rpc api server on http://%s:%s" % (args.host, args.port_rpc)) api_server_rpc = JsonRpcApi(args.port_rpc, wallet=wallet) endpoint_rpc = "tcp:port={0}:interface={1}".format(args.port_rpc, args.host) endpoints.serverFromString(reactor, endpoint_rpc).listen(Site(api_server_rpc.app.resource())) # reactor.listenTCP(int(args.port_rpc), server.Site(api_server_rpc)) # api_server_rpc.app.run(args.host, args.port_rpc) if args.port_rest: logger.info("Starting REST api server on http://%s:%s" % (args.host, args.port_rest)) api_server_rest = RestApi() endpoint_rest = "tcp:port={0}:interface={1}".format(args.port_rest, args.host) endpoints.serverFromString(reactor, endpoint_rest).listen(Site(api_server_rest.app.resource())) # api_server_rest.app.run(args.host, args.port_rest) reactor.run() # After the reactor is stopped, gracefully shutdown the database. logger.info("Closing databases...") NotificationDB.close() Blockchain.Default().Dispose() NodeLeader.Instance().Shutdown() if wallet: wallet.Close()
def __enter__(self): set_global_log_level(self.desired_level) globalLogPublisher.addObserver(self._got_log) return self
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument( '--loglevel', default="info", choices=['none', 'error', 'warn', 'info', 'debug', 'trace'], help='Initial log level.') parser.add_argument('-c', '--cbdir', type=six.text_type, help="Crossbar.io node directory (required).") parser.add_argument('-n', '--node', type=six.text_type, help='Crossbar.io node ID (required).') parser.add_argument('-w', '--worker', type=six.text_type, help='Crossbar.io worker ID (required).') parser.add_argument('-r', '--realm', type=six.text_type, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container', 'websocket-testee'], help='Worker type (required).') parser.add_argument('--title', type=six.text_type, default=None, help='Worker process title to set (optional).') options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware, _stderr from crossbar._logging import make_logger, start_logging, set_global_log_level from twisted.logger import globalLogPublisher # Set the global log level set_global_log_level(options.loglevel) log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=_stderr) _stderr.flush() flo = make_JSON_observer(_stderr) globalLogPublisher.addObserver(flo) start_logging() # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) from twisted.python.reflect import qual log.info("Worker process starting ({python}-{reactor}) ..", python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]', 'websocket-testee': 'crossbar-worker [websocket-testee]' } setproctitle.setproctitle( WORKER_TYPE_TO_TITLE[options.type].strip()) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession from crossbar.worker.testee import WebSocketTesteeWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession, 'websocket-testee': WebSocketTesteeWorkerSession } from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False) transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {}".format(e)) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def start_sentry_logging(cls, dsn: str): _SentryInitGuard.init(dsn) globalLogPublisher.addObserver(sentry_observer)
def run(options, reactor=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # eg: crossbar.worker.container.ContainerWorkerSession l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Started {worker_title} worker "{worker_id}" on node "{node_id}" [{klass} / {python}-{reactor}]', worker_title=klass.WORKER_TITLE, klass=options.klass, node_id=options.node, worker_id=options.worker, pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = klass # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def _startlog(options, reactor): """ Start the logging in a way that all the subcommands can use it. """ from twisted.logger import globalLogPublisher from txaio import start_logging, set_global_log_level loglevel = getattr(options, "loglevel", "info") logformat = getattr(options, "logformat", "none") colour = getattr(options, "colour", "auto") set_global_log_level(loglevel) # The log observers (things that print to stderr, file, etc) observers = [] if getattr(options, "logtofile", False): # We want to log to a file from crossbar._logging import make_logfile_observer if not options.logdir: logdir = options.cbdir else: logdir = options.logdir logfile = os.path.join(logdir, "node.log") if loglevel in ["error", "warn", "info"]: show_source = False else: show_source = True observers.append(make_logfile_observer(logfile, show_source)) else: # We want to log to stdout/stderr. from crossbar._logging import make_stdout_observer from crossbar._logging import make_stderr_observer from crossbar._logging import LogLevel if colour == "auto": if sys.__stdout__.isatty(): colour = True else: colour = False elif colour == "true": colour = True else: colour = False if loglevel == "none": # Do no logging! pass elif loglevel in ["error", "warn", "info"]: # Print info to stdout, warn+ to stderr observers.append(make_stdout_observer(show_source=False, format=logformat, colour=colour)) observers.append(make_stderr_observer(show_source=False, format=logformat, colour=colour)) elif loglevel == "debug": # Print debug+info to stdout, warn+ to stderr, with the class # source observers.append(make_stdout_observer(show_source=True, levels=(LogLevel.info, LogLevel.debug), format=logformat, colour=colour)) observers.append(make_stderr_observer(show_source=True, format=logformat, colour=colour)) elif loglevel == "trace": # Print trace+, with the class source observers.append(make_stdout_observer(show_source=True, levels=(LogLevel.info, LogLevel.debug), format=logformat, trace=True, colour=colour)) observers.append(make_stderr_observer(show_source=True, format=logformat, colour=colour)) else: assert False, "Shouldn't ever get here." for observer in observers: globalLogPublisher.addObserver(observer) # Make sure that it goes away reactor.addSystemEventTrigger('after', 'shutdown', globalLogPublisher.removeObserver, observer) # Actually start the logger. start_logging(None, loglevel)
import sys from twisted.logger import (Logger, LogLevel, LogLevelFilterPredicate, FilteringLogObserver, textFileLogObserver, globalLogPublisher) __all__ = ["logger"] predicate = LogLevelFilterPredicate(defaultLogLevel=LogLevel.debug) observer = FilteringLogObserver(textFileLogObserver(sys.stdout), [predicate]) observer._encoding = "utf-8" globalLogPublisher.addObserver(observer) logger = Logger()
def test_invalid_workers_tolerance(testerchain, test_registry, blockchain_ursulas, agency, idle_staker, token_economics, ursula_decentralized_test_config ): # # Setup # lonely_blockchain_learner, blockchain_teacher, unsigned, *the_others = list(blockchain_ursulas) _, staking_agent, _ = agency warnings = [] def warning_trapper(event): if event['log_level'] == LogLevel.warn: warnings.append(event) # We start with an "idle_staker" (i.e., no tokens in StakingEscrow) assert 0 == staking_agent.owned_tokens(idle_staker.checksum_address) # Now let's create an active worker for this staker. # First, stake something (e.g. the bare minimum) amount = token_economics.minimum_allowed_locked periods = token_economics.minimum_locked_periods # Mock Powerup consumption (Staker) testerchain.transacting_power = TransactingPower(account=idle_staker.checksum_address) idle_staker.initialize_stake(amount=amount, lock_periods=periods) # Stake starts next period (or else signature validation will fail) testerchain.time_travel(periods=1) idle_staker.stake_tracker.refresh() # We create an active worker node for this staker worker = make_ursula_for_staker(staker=idle_staker, worker_address=testerchain.unassigned_accounts[-1], ursula_config=ursula_decentralized_test_config, blockchain=testerchain, commit_now=True, ursulas_to_learn_about=None) # Since we made a commitment, we need to advance one period testerchain.time_travel(periods=1) # The worker is valid and can be verified (even with the force option) worker.verify_node(force=True, network_middleware=MockRestMiddleware(), certificate_filepath="quietorl") # In particular, we know that it's bonded to a staker who is really staking. assert worker._worker_is_bonded_to_staker(registry=test_registry) assert worker._staker_is_really_staking(registry=test_registry) # OK. Now we learn about this worker. lonely_blockchain_learner.remember_node(worker) # The worker already committed one period before. Let's commit to the remaining 29. for i in range(29): worker.commit_to_next_period() testerchain.time_travel(periods=1) # The stake period has ended, and the staker wants her tokens back ("when lambo?"). # She withdraws up to the last penny (well, last nunit, actually). # Mock Powerup consumption (Staker) testerchain.transacting_power = TransactingPower(account=idle_staker.checksum_address) idle_staker.mint() testerchain.time_travel(periods=1) i_want_it_all = staking_agent.owned_tokens(idle_staker.checksum_address) idle_staker.withdraw(i_want_it_all) # OK...so...the staker is not staking anymore ... assert 0 == staking_agent.owned_tokens(idle_staker.checksum_address) # ... but the worker node still is "verified" (since we're not forcing on-chain verification) worker.verify_node(network_middleware=MockRestMiddleware(), certificate_filepath="quietorl") # If we force, on-chain verification, the worker is of course not verified with pytest.raises(worker.NotStaking): worker.verify_node(force=True, network_middleware=MockRestMiddleware(), certificate_filepath="quietorl") # Let's learn from this invalid node lonely_blockchain_learner._current_teacher_node = worker globalLogPublisher.addObserver(warning_trapper) lonely_blockchain_learner.learn_from_teacher_node() # lonely_blockchain_learner.remember_node(worker) # The same problem occurs if we directly try to remember this node globalLogPublisher.removeObserver(warning_trapper) # TODO: What should we really check here? (#1075) assert len(warnings) == 1 warning = warnings[-1]['log_format'] assert str(worker) in warning assert "no active stakes" in warning # TODO: Cleanup logging templates assert worker not in lonely_blockchain_learner.known_nodes
def create_application(args=None, twistd_user=None, log_group=None): home_dir = util.get_home_dir() os.chdir(home_dir) is_logging_insecure = False # parse command-line args, if appropriate primary_only_time = None if args: option_parser = argparse.ArgumentParser() option_parser.add_argument( "--primary-only", type=int, nargs="?", help= "This option disables secondary authentication for the specified number of minutes (default 60)", default=None, const=60, ) option_parser.add_argument( "--logging-insecure", action="store_true", help= "This option enables debug, and prints logs containing passwords and possibly other secrets.", default=False, ) options = option_parser.parse_args() is_logging_insecure = options.logging_insecure primary_only_time = options.primary_only config_filename = os.path.join("conf", "authproxy.cfg") configuration = config_provider.get_config(config_filename) if primary_only_time is not None: if primary_only_time > 240: print( "Primary only mode can only be enabled for a maximum of 4 hours (240 minutes)" ) sys.exit(2) else: PrimaryOnlyManager.enable_primary_only(primary_only_time) main_config = configuration.get_main_section_config() if main_config: log.msg("Main Configuration:") log.config(main_config) fips_mode = main_config.get_bool("fips_mode", False) if fips_mode: fips_manager.enable() # Set up our observers if is_logging_insecure: observers = [textFileLogObserver(sys.stdout)] else: observers = log_observation.get_observers(main_config, twistd_user, log_group) for observer in observers: globalLogPublisher.addObserver(observer) # Global debug mode if is_logging_insecure: debug_mode = True else: debug_mode = main_config.get_bool("debug", False) http.set_debug(debug_mode) http.set_is_logging_insecure(is_logging_insecure) # Create main application. application = Application("duoauthproxy") LogReadyService().setServiceParent(application) fips_mode = fips_manager.status() if fips_mode: log.msg("FIPS mode {0} is enabled with {1}".format( fips_mode, fips_manager.get_openssl_version())) else: log.msg("FIPS mode is not enabled") # get ca certs file http_ca_certs_file = main_config.get_str("http_ca_certs_file", "") if http_ca_certs_file: http_ca_certs_file = util.resolve_file_path(http_ca_certs_file) else: http_ca_certs_file = os.path.join("conf", const.DEFAULT_HTTP_CERTS_FILE) # read ca certs if not os.path.isfile(http_ca_certs_file): http_ca_certs_file = os.path.join("conf", http_ca_certs_file) with open(http_ca_certs_file, "r") as bundle_fp: http.set_ca_certs(ssl_verify.load_ca_bundle(bundle_fp)) # get proxy settings http_proxy_host = main_config.get_str("http_proxy_host", "") http_proxy_port = main_config.get_int("http_proxy_port", 80) if http_proxy_host: http.set_proxy(http_proxy_host, http_proxy_port) sections = section.parse_sections(configuration, is_logging_insecure) module_factory = section.ModuleFactory(sections, application) modules_by_type = module_factory.make_modules() if not any(modules_by_type.values()): raise config_error.ConfigError("No integrations in config file.") # Setup forwarding/server pairs by port for port, interface in modules_by_type.get("server", []): server_networks = {} server_names = {} for section_name, server_module, server_config in modules_by_type[ "server"][(port, interface)]: client_name = configuration.get_section_client(section_name) if not client_name: if server_module.Module.no_client: modules_by_type["client"]["no_client"] = None client_name = "no_client" else: raise config_error.ConfigError( 'Neither module %s or main has "client" value' % section_name) if section_name.startswith( "ldap_server_auto" ) and not client_name.startswith("ad_client"): raise config_error.ConfigError( "ad_client is required by ldap_server_auto. No ad_client found in config file. " ) if client_name != "radius_client" and server_config.get_str( "pass_through_attr_names", ""): raise config_error.ConfigError( "Can only pass through radius attributes if using a radius client" ) server_instance = server_module.Module( server_config, modules_by_type["client"][client_name], section_name) server_instance.setServiceParent(application) if section_name.startswith("radius_server_"): server_networks[server_instance] = parse_radius_secrets( server_config).keys() server_names[server_instance] = section_name if server_names: forward_module = forward_serv forward_instance = forward_module.Module( port=port, servers=server_networks, server_names=server_names, interface=interface, debug=debug_mode, ) forward_instance.setServiceParent(application) # set user-agent sections = ",".join(sorted(set(configuration.list_sections()))) user_agent = "duoauthproxy/{0} ({1}; Python{2}; {3})".format( get_version(), platform.platform(), platform.python_version(), sections) http.set_user_agent(user_agent) # Authproxy uses globalLogPublisher to emit events. Defining a no-op emitter will squelch the creation # of the unwatned twistd default logging mechanisms. def no_op_emitter(eventDict): pass application.setComponent(ILogObserver, no_op_emitter) return application
from .defaults import DEFAULT_LOG_FILE from twisted.logger import ( ILogObserver, jsonFileLogObserver, FilteringLogObserver, LogLevelFilterPredicate, LogLevel, globalLogPublisher ) from zope.interface import provider import io @provider(ILogObserver) def hendrixObserver(path=DEFAULT_LOG_FILE, log_level=LogLevel.warn): json_observer = jsonFileLogObserver( io.open(path, 'a') ) return FilteringLogObserver( json_observer, [LogLevelFilterPredicate(log_level), ] ) globalLogPublisher.addObserver(hendrixObserver(log_level=LogLevel.debug))
def __attrs_post_init__(self) -> None: globalLogPublisher.addObserver(self.storeObserver)
def _run_command_exec_worker(options, reactor=None, personality=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes MEASURING_COVERAGE = False if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage except ImportError: pass else: # The following will read the environment variable COVERAGE_PROCESS_START, # and that should be set to the .coveragerc file: # # export COVERAGE_PROCESS_START=${PWD}/.coveragerc # coverage.process_startup() MEASURING_COVERAGE = True # we use an Autobahn utility to import the "best" available Twisted reactor from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # make sure logging to something else than stdio is setup _first_ from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # get personality klass, eg "crossbar.personality.Personality" l = options.personality.split('.') personality_module, personality_klass = '.'.join(l[:-1]), l[-1] # now load the personality module and class _mod = importlib.import_module(personality_module) Personality = getattr(_mod, personality_klass) # get worker klass, eg "crossbar.worker.container.ContainerController" l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Starting worker "{worker_id}" for node "{node_id}" with personality "{personality}" {worker_class}', worker_id=options.worker, node_id=options.node, personality=Personality.NAME, worker_class=hltype(klass), ) log.info( 'Running as PID {pid} on {python}-{reactor}', pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) if MEASURING_COVERAGE: log.info(hl('Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True), coverage_version=coverage.__version__) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format(options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle( 'crossbar-worker [{}]'.format(options.klass) ) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass try: # define a WAMP application session factory # from autobahn.wamp.types import ComponentConfig def make_session(): session_config = ComponentConfig(realm=options.realm, extra=options) session = klass(config=session_config, reactor=reactor, personality=Personality) return session # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() vmprof.disable() else: log.info(hl('Entering event reactor ...', color='cyan', bold=True)) reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def test_emit_warning_upon_new_version(ursula_federated_test_config, caplog): nodes = make_federated_ursulas(ursula_config=ursula_federated_test_config, quantity=3, know_each_other=False) teacher, learner, new_node = nodes learner.remember_node(teacher) teacher.remember_node(learner) teacher.remember_node(new_node) new_node.TEACHER_VERSION = learner.LEARNER_VERSION + 1 warnings = [] def warning_trapper(event): if event['log_level'] == LogLevel.warn: warnings.append(event) globalLogPublisher.addObserver(warning_trapper) learner.learn_from_teacher_node() assert len(warnings) == 1 assert warnings[0]['log_format'] == learner.unknown_version_message.format( new_node, new_node.TEACHER_VERSION, learner.LEARNER_VERSION) # Now let's go a little further: make the version totally unrecognizable. # First, there's enough garbage to at least scrape a potential checksum address fleet_snapshot = os.urandom(32 + 4) random_bytes = os.urandom(50) # lots of garbage in here future_version = learner.LEARNER_VERSION + 42 version_bytes = future_version.to_bytes(2, byteorder="big") crazy_bytes = fleet_snapshot + VariableLengthBytestring(version_bytes + random_bytes) signed_crazy_bytes = bytes(teacher.stamp(crazy_bytes)) Response = namedtuple("MockResponse", ("content", "status_code")) response = Response(content=signed_crazy_bytes + crazy_bytes, status_code=200) learner._current_teacher_node = teacher learner.network_middleware.get_nodes_via_rest = lambda *args, **kwargs: response learner.learn_from_teacher_node() # If you really try, you can read a node representation from the garbage accidental_checksum = to_checksum_address(random_bytes[:20]) accidental_nickname = nickname_from_seed(accidental_checksum)[0] accidental_node_repr = Character._display_name_template.format( "Ursula", accidental_nickname, accidental_checksum) assert len(warnings) == 2 assert warnings[1]['log_format'] == learner.unknown_version_message.format( accidental_node_repr, future_version, learner.LEARNER_VERSION) # This time, however, there's not enough garbage to assume there's a checksum address... random_bytes = os.urandom(2) crazy_bytes = fleet_snapshot + VariableLengthBytestring(version_bytes + random_bytes) signed_crazy_bytes = bytes(teacher.stamp(crazy_bytes)) response = Response(content=signed_crazy_bytes + crazy_bytes, status_code=200) learner._current_teacher_node = teacher learner.learn_from_teacher_node() assert len(warnings) == 3 # ...so this time we get a "really unknown version message" assert warnings[2][ 'log_format'] == learner.really_unknown_version_message.format( future_version, learner.LEARNER_VERSION) globalLogPublisher.removeObserver(warning_trapper)
def activate(cls, host): cls.producer = KafkaProducer(bootstrap_servers=host) globalLogPublisher.addObserver(kafka_observer)
def _run_command_exec_worker(options, reactor=None, personality=None): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html#measuring-sub-processes MEASURING_COVERAGE = False if 'COVERAGE_PROCESS_START' in os.environ: try: import coverage except ImportError: pass else: # The following will read the environment variable COVERAGE_PROCESS_START, # and that should be set to the .coveragerc file: # # export COVERAGE_PROCESS_START=${PWD}/.coveragerc # coverage.process_startup() MEASURING_COVERAGE = True # we use an Autobahn utility to import the "best" available Twisted reactor from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) # make sure logging to something else than stdio is setup _first_ from crossbar._logging import make_JSON_observer, cb_logging_aware from txaio import make_logger, start_logging from twisted.logger import globalLogPublisher from twisted.python.reflect import qual log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=sys.__stderr__) sys.__stderr__.flush() flo = make_JSON_observer(sys.__stderr__) globalLogPublisher.addObserver(flo) # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # actually begin logging start_logging(None, options.loglevel) # get personality klass, eg "crossbar.personality.Personality" l = options.personality.split('.') personality_module, personality_klass = '.'.join(l[:-1]), l[-1] # now load the personality module and class _mod = importlib.import_module(personality_module) Personality = getattr(_mod, personality_klass) # get worker klass, eg "crossbar.worker.container.ContainerController" l = options.klass.split('.') worker_module, worker_klass = '.'.join(l[:-1]), l[-1] # now load the worker module and class _mod = importlib.import_module(worker_module) klass = getattr(_mod, worker_klass) log.info( 'Starting worker "{worker_id}" for node "{node_id}" on realm "{realm}" with personality "{personality}" {worker_class}', worker_id=options.worker, node_id=options.node, realm=options.realm, personality=Personality.NAME, worker_class=hltype(klass), ) log.info( 'Running as PID {pid} on {python}-{reactor}', pid=os.getpid(), python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1], ) if MEASURING_COVERAGE: log.info(hl( 'Code coverage measurements enabled (coverage={coverage_version}).', color='green', bold=True), coverage_version=coverage.__version__) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) # node directory # options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) # set process title if requested to # try: import setproctitle except ImportError: log.debug( "Could not set worker process title (setproctitle not installed)") else: if options.title: setproctitle.setproctitle(options.title) else: setproctitle.setproctitle('crossbar-worker [{}]'.format( options.klass)) from twisted.internet.error import ConnectionDone from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): # the behavior here differs slightly whether we're shutting down orderly # or shutting down because of "issues" if isinstance(reason.value, ConnectionDone): was_clean = True else: was_clean = False try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) if was_clean: log.info("Connection to node controller closed cleanly") else: log.warn("Connection to node controller lost: {reason}", reason=reason) # give the WAMP transport a change to do it's thing WampWebSocketServerProtocol.connectionLost(self, reason) except: # we're in the process of shutting down .. so ignore .. pass finally: # after the connection to the node controller is gone, # the worker is "orphane", and should exit # determine process exit code if was_clean: exit_code = 0 else: exit_code = 1 # exit the whole worker process when the reactor has stopped reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code) # stop the reactor try: reactor.stop() except ReactorNotRunning: pass # if vmprof global profiling is enabled via command line option, this will carry # the file where vmprof writes its profile data _vm_prof = { # need to put this into a dict, since FDs are ints, and python closures can't # write to this otherwise 'outfd': None } # https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IReactorCore.html # Each "system event" in Twisted, such as 'startup', 'shutdown', and 'persist', has 3 phases: # 'before', 'during', and 'after' (in that order, of course). These events will be fired # internally by the Reactor. def before_reactor_started(): term_print('CROSSBAR[{}]:REACTOR_STARTING'.format(options.worker)) def after_reactor_started(): term_print('CROSSBAR[{}]:REACTOR_STARTED'.format(options.worker)) if options.vmprof: outfn = os.path.join( options.cbdir, '.vmprof-worker-{}-{}.dat'.format(options.worker, os.getpid())) _vm_prof['outfd'] = os.open(outfn, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(_vm_prof['outfd'], period=0.01) term_print('CROSSBAR[{}]:VMPROF_ENABLED:{}'.format( options.worker, outfn)) def before_reactor_stopped(): term_print('CROSSBAR[{}]:REACTOR_STOPPING'.format(options.worker)) if options.vmprof and _vm_prof['outfd']: vmprof.disable() term_print('CROSSBAR[{}]:VMPROF_DISABLED'.format(options.worker)) def after_reactor_stopped(): # FIXME: we are indeed reaching this line, however, # the log output does not work (it also doesnt work using # plain old print). Dunno why. # my theory about this issue is: by the time this line # is reached, Twisted has already closed the stdout/stderr # pipes. hence we do an evil trick: we directly write to # the process' controlling terminal # https://unix.stackexchange.com/a/91716/52500 term_print('CROSSBAR[{}]:REACTOR_STOPPED'.format(options.worker)) reactor.addSystemEventTrigger('before', 'startup', before_reactor_started) reactor.addSystemEventTrigger('after', 'startup', after_reactor_started) reactor.addSystemEventTrigger('before', 'shutdown', before_reactor_stopped) reactor.addSystemEventTrigger('after', 'shutdown', after_reactor_stopped) try: # define a WAMP application session factory # from autobahn.wamp.types import ComponentConfig def make_session(): session_config = ComponentConfig(realm=options.realm, extra=options) session = klass(config=session_config, reactor=reactor, personality=Personality) return session # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(make_session, u'ws://localhost') transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # log.info(hl('Entering event reactor ...', color='green', bold=True)) reactor.run() except Exception as e: log.info("Unhandled exception: {e}", e=e) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def _start_logging(options, reactor): """ Start the logging in a way that all the subcommands can use it. """ loglevel = getattr(options, "loglevel", "info") logformat = getattr(options, "logformat", "none") color = getattr(options, "color", "auto") set_global_log_level(loglevel) # The log observers (things that print to stderr, file, etc) observers = [] if getattr(options, "logtofile", False): # We want to log to a file if not options.logdir: logdir = options.cbdir else: logdir = options.logdir logfile = os.path.join(logdir, "node.log") if loglevel in ["error", "warn", "info"]: show_source = False else: show_source = True observers.append(make_logfile_observer(logfile, show_source)) else: # We want to log to stdout/stderr. if color == "auto": if sys.__stdout__.isatty(): color = True else: color = False elif color == "true": color = True else: color = False if loglevel == "none": # Do no logging! pass elif loglevel in ["error", "warn", "info"]: # Print info to stdout, warn+ to stderr observers.append( make_stdout_observer(show_source=False, format=logformat, color=color)) observers.append( make_stderr_observer(show_source=False, format=logformat, color=color)) elif loglevel == "debug": # Print debug+info to stdout, warn+ to stderr, with the class # source observers.append( make_stdout_observer(show_source=True, levels=(LogLevel.info, LogLevel.debug), format=logformat, color=color)) observers.append( make_stderr_observer(show_source=True, format=logformat, color=color)) elif loglevel == "trace": # Print trace+, with the class source observers.append( make_stdout_observer(show_source=True, levels=(LogLevel.info, LogLevel.debug), format=logformat, trace=True, color=color)) observers.append( make_stderr_observer(show_source=True, format=logformat, color=color)) else: assert False, "Shouldn't ever get here." for observer in observers: globalLogPublisher.addObserver(observer) # Make sure that it goes away reactor.addSystemEventTrigger('after', 'shutdown', globalLogPublisher.removeObserver, observer) # Actually start the logger. start_logging(None, loglevel)
def enable_twisted_log_observer(): globalLogPublisher.addObserver(twisted_log_observer)
def startService(self): self.stdlib_cleanup = _stdlib_logging_to_eliot_configuration(getLogger()) self.twisted_observer = _TwistedLoggerToEliotObserver() globalLogPublisher.addObserver(self.twisted_observer) add_destinations(*self.destinations) return Service.startService(self)
def requestAvatar(self, avatarId, mind, *interfaces): if IFTPShell in interfaces: return (IFTPShell, FTPShell(filepath.FilePath(VIDEOROOT)), lambda: None) if IMessageDelivery in interfaces: return (IMessageDelivery, ConsoleMessageDelivery(), lambda: None) raise NotImplementedError( "Unable to provide avatar for interfaces provided ({})".format( interfaces)) log = Logger("amcrest") predicate = LogLevelFilterPredicate(LogLevel.warn) predicate.setLogLevelForNamespace("amcrest", LogLevel.debug) globalLogPublisher.addObserver( FilteringLogObserver(textFileLogObserver(sys.stderr), (predicate, ))) portal = Portal(AmcrestRealm(), [ FilePasswordDB( os.path.join(os.path.dirname(os.path.abspath(__file__)), "passwords.txt")) ]) ftpfactory = FTPFactory(portal) ftpfactory.allowAnonymous = False ftpfactory.protocol = MonitoredFTP hafactory = HAFactory() reactor.listenTCP(2121, ftpfactory) reactor.listenTCP(2525, ConsoleSMTPFactory(portal))
def apply_logging(self): # We're using twisted logging only for IO. from twisted.logger import FileLogObserver from twisted.logger import Logger, LogLevel, globalLogPublisher LOGLEVEL_TWISTED_MAP = { logging.DEBUG: LogLevel.debug, logging.INFO: LogLevel.info, logging.WARN: LogLevel.warn, logging.ERROR: LogLevel.error, logging.CRITICAL: LogLevel.critical, } TWISTED_LOGLEVEL_MAP = {v: k for k, v in LOGLEVEL_TWISTED_MAP.items()} loggers = {} config = self class TwistedHandler(logging.Handler): if config.log_rss: if _meminfo is None: def _modify_record(self, record): record.msg = '[psutil?] %s' % record.msg else: def _modify_record(self, record): meminfo = _meminfo() record.msg = '[%.2f] %s' % (meminfo.rss / 1024.0 ** 2, record.msg) else: def _modify_record(self, record): pass def emit(self, record): assert isinstance(record, logging.LogRecord) record.l = LOGLEVEL_MAP_ABB.get(record.levelno, "?") self._modify_record(record) _logger = loggers.get(record.name, None) if _logger is None: _logger = loggers[record.name] = Logger(record.name) t = self.format(record) if six.PY2 and isinstance(t, str): # Libraries outside of our control can throw exceptions with # a "str with known encoding", which can cause issues down # the logging pipeline. Her t = t.decode('utf8', errors='replace') _logger.emit(LOGLEVEL_TWISTED_MAP[record.levelno], log_text=t) if self.logger_dest is not None: from twisted.python.logfile import DailyLogFile class DailyLogWithLeadingZero(DailyLogFile): def suffix(self, tupledate): # this closely imitates the same function from parent class try: return '-'.join(("%02d" % i for i in tupledate)) except: # try taking a float unixtime return '-'.join(("%02d" % i for i in self.toDate(tupledate))) self.logger_dest = abspath(self.logger_dest) if access(dirname(self.logger_dest), os.R_OK | os.W_OK): log_dest = DailyLogWithLeadingZero \ .fromFullPath(self.logger_dest) else: Logger().warn("%r is not accessible. We need rwx on it to " "rotate logs." % dirname(self.logger_dest)) log_dest = open(self.logger_dest, 'w+') formatter = logging.Formatter(self.LOGGING_PROD_FORMAT) else: formatter = logging.Formatter(self.LOGGING_DEVEL_FORMAT) log_dest = sys.stdout try: import colorama colorama.init() logger.debug("colorama loaded.") except Exception as e: logger.debug("coloarama not loaded: %r" % e) def record_as_string(record): if 'log_failure' in record: failure = record['log_failure'] try: s = pformat(vars(failure.value)) except TypeError: # vars() argument must have __dict__ attribute s = repr(failure.value) return "%s: %s" % (failure.type, s) if 'log_text' in record: return record['log_text'] + "\n" if 'log_format' in record: level = record.get('log_level', LogLevel.debug) level = LOGLEVEL_MAP_ABB[TWISTED_LOGLEVEL_MAP[level]] text = record['log_format'].format(**record) + "\n" ns = record.get('log_namespace', "???") lineno = 0 record = logging.LogRecord('?', level, ns, lineno, text, None, None) record.l = level record.module = ns.split('.')[-2] return formatter.format(record) if 'log_io' in record: return record['log_io'] + "\n" if 'message' in record: return record['message'] + "\n" return pformat(record) observer = FileLogObserver(log_dest, record_as_string) globalLogPublisher.addObserver(observer) self._clear_other_observers(globalLogPublisher, observer) handler = TwistedHandler() handler.setFormatter(formatter) logging.getLogger().addHandler(handler) self.pre_logging_apply() for l in self._loggers or []: l.apply() return self
def doctor_decrypt(hash_key): globalLogPublisher.addObserver(SimpleObserver()) SEEDNODE_URL = 'localhost:11501' TEMP_DOCTOR_DIR = "{}/doctor-files".format( os.path.dirname(os.path.abspath(__file__))) shutil.rmtree(TEMP_DOCTOR_DIR, ignore_errors=True) ursula = Ursula.from_seed_and_stake_info(seed_uri=SEEDNODE_URL, federated_only=True, minimum_stake=0) from doctor_keys import get_doctor_privkeys doctor_keys = get_doctor_privkeys() bob_enc_keypair = DecryptingKeypair(private_key=doctor_keys["enc"]) bob_sig_keypair = SigningKeypair(private_key=doctor_keys["sig"]) enc_power = DecryptingPower(keypair=bob_enc_keypair) sig_power = SigningPower(keypair=bob_sig_keypair) power_ups = [enc_power, sig_power] print("Creating the Doctor ...") doctor = Bob( is_me=True, federated_only=True, crypto_power_ups=power_ups, start_learning_now=True, abort_on_learning_error=True, known_nodes=[ursula], save_metadata=False, network_middleware=RestMiddleware(), ) print("Doctor = ", doctor) with open("policy-metadata.json", 'r') as f: policy_data = json.load(f) policy_pubkey = UmbralPublicKey.from_bytes( bytes.fromhex(policy_data["policy_pubkey"])) alices_sig_pubkey = UmbralPublicKey.from_bytes( bytes.fromhex(policy_data["alice_sig_pubkey"])) label = policy_data["label"].encode() print("The Doctor joins policy for label '{}'".format( label.decode("utf-8"))) doctor.join_policy(label, alices_sig_pubkey) ipfs_api = ipfsapi.connect() file = ipfs_api.get(hash_key) print(file) os.rename(hash_key, 'patient_details.msgpack') data = msgpack.load(open("patient_details.msgpack", "rb"), raw=False) message_kits = (UmbralMessageKit.from_bytes(k) for k in data['kits']) data_source = DataSource.from_public_keys( policy_public_key=policy_pubkey, datasource_public_key=data['data_source'], label=label) complete_message = [] for message_kit in message_kits: print(message_kit) try: start = timer() retrieved_plaintexts = doctor.retrieve( message_kit=message_kit, data_source=data_source, alice_verifying_key=alices_sig_pubkey) end = timer() plaintext = msgpack.loads(retrieved_plaintexts[0], raw=False) complete_message.append(plaintext) print(plaintext) #with open("details.json", "w") as write_file: # json.dump(plaintext, write_file) except Exception as e: traceback.print_exc() with open("details.json", "w") as write_file: json.dump(complete_message, write_file) return complete_message
def start_json_file_logging(cls): globalLogPublisher.addObserver(get_json_file_observer())
output.send(vals) def inputActions(inputs): for input in inputs: input.fetch(outputActions) def shutdown(): print("Shutdown") global SolaxModbusInverters for inverter in SolaxModbusInverters: inverter.shutdown() globalLogPublisher.addObserver(analyze) with open("config.toml") as conffile: global config config = toml.loads(conffile.read()) # pp.pprint(config) global outputs outputs = [] global SolaxModbusInverters SolaxModbusInverters = [] if 'emoncms' in config: print("Setting up EmonCMS")
def start_console_logging(cls): globalLogPublisher.addObserver(console_observer)
def run_doc(): globalLogPublisher.addObserver(SimpleObserver()) ###################### # Boring setup stuff # ###################### SEEDNODE_URL = 'localhost:11501' # TODO: path joins? TEMP_DOCTOR_DIR = "{}/doctor-files".format( os.path.dirname(os.path.abspath(__file__))) # Remove previous demo files and create new ones shutil.rmtree(TEMP_DOCTOR_DIR, ignore_errors=True) ursula = Ursula.from_seed_and_stake_info(seed_uri=SEEDNODE_URL, federated_only=True, minimum_stake=0) # To create a Bob, we need the doctor's private keys previously generated. doctor_keys = get_doctor_privkeys() bob_enc_keypair = DecryptingKeypair(private_key=doctor_keys["enc"]) bob_sig_keypair = SigningKeypair(private_key=doctor_keys["sig"]) enc_power = DecryptingPower(keypair=bob_enc_keypair) sig_power = SigningPower(keypair=bob_sig_keypair) power_ups = [enc_power, sig_power] print("Creating the Doctor ...") doctor = Bob( is_me=True, federated_only=True, crypto_power_ups=power_ups, start_learning_now=True, abort_on_learning_error=True, known_nodes=[ursula], save_metadata=False, network_middleware=RestMiddleware(), ) print("Doctor = ", doctor) # Let's join the policy generated by Alicia. We just need some info about it. with open("policy-metadata.json", 'r') as f: policy_data = json.load(f) policy_pubkey = UmbralPublicKey.from_bytes( bytes.fromhex(policy_data["policy_pubkey"])) alices_sig_pubkey = UmbralPublicKey.from_bytes( bytes.fromhex(policy_data["alice_sig_pubkey"])) label = policy_data["label"].encode() print("The Doctor joins policy for label '{}'".format( label.decode("utf-8"))) doctor.join_policy(label, alices_sig_pubkey) # Now that the Doctor joined the policy in the NuCypher network, # he can retrieve encrypted data which he can decrypt with his private key. # But first we need some encrypted data! # Let's read the file produced by the heart monitor and unpack the MessageKits, # which are the individual ciphertexts. data = msgpack.load(open("heart_data.msgpack", "rb"), raw=False) message_kits = (UmbralMessageKit.from_bytes(k) for k in data['kits']) # The doctor also needs to create a view of the Data Source from its public keys data_source = Enrico.from_public_keys({SigningPower: data['data_source']}, policy_encrypting_key=policy_pubkey) # Now he can ask the NuCypher network to get a re-encrypted version of each MessageKit. for message_kit in message_kits: try: start = timer() retrieved_plaintexts = doctor.retrieve( label=label, message_kit=message_kit, data_source=data_source, alice_verifying_key=alices_sig_pubkey) end = timer() plaintext = msgpack.loads(retrieved_plaintexts[0], raw=False) # Now we can get the heart rate and the associated timestamp, # generated by the heart rate monitor. heart_rate = plaintext['heart_rate'] timestamp = maya.MayaDT(plaintext['timestamp']) # This code block simply pretty prints the heart rate info terminal_size = shutil.get_terminal_size().columns max_width = min(terminal_size, 120) columns = max_width - 12 - 27 scale = columns / 40 scaled_heart_rate = int(scale * (heart_rate - 60)) retrieval_time = "Retrieval time: {:8.2f} ms".format(1000 * (end - start)) line = ("-" * scaled_heart_rate) + "❤︎ ({} BPM)".format(heart_rate) line = line.ljust(max_width - 27, " ") + retrieval_time print(line) except Exception as e: # We just want to know what went wrong and continue the demo traceback.print_exc()
threadpool=thpool) is_logging = True is_log2file = False if is_logging: if is_log2file: DEFAULT_LOG_FILE = os.path.join(settings.BASE_DIR, 'default-hendrix.log') @provider(ILogObserver) def FileObserver(path=DEFAULT_LOG_FILE, log_level=LogLevel.warn): file_observer = textFileLogObserver(io.open(path, 'at')) return FilteringLogObserver(file_observer, [ LogLevelFilterPredicate(log_level), ]) log_path = os.path.join(settings.BASE_DIR, 'hendrix-debug.log') globalLogPublisher.addObserver( FileObserver(path=log_path, log_level=LogLevel.debug)) else: @provider(ILogObserver) def ConsoleObserver(log_level=LogLevel.warn): console_observer = textFileLogObserver(sys.stdout) return FilteringLogObserver(console_observer, [ LogLevelFilterPredicate(log_level), ]) globalLogPublisher.addObserver( ConsoleObserver(log_level=LogLevel.debug)) deployer.run()
from twisted.internet.endpoints import serverFromString from twisted.internet.defer import gatherResults, inlineCallbacks, Deferred from psycopg2.extras import NamedTupleCursor from txpostgres import txpostgres from txchatexample.util import ConnectionPool from txchatexample.web import makeEntryPoint @inlineCallbacks def main(reactor, server='tcp:8000'): pool = ConnectionPool(dbname='asdf') conn = txpostgres.Connection() yield gatherResults([ pool.start(), conn.connect(dbname='asdf', cursor_factory=NamedTupleCursor) ]) res = yield makeEntryPoint(pool, conn) site = Site(res) ep = serverFromString(reactor, server) ep.listen(site) yield Deferred() # wait forever if __name__ == '__main__': globalLogPublisher.addObserver(textFileLogObserver(sys.stdout)) react(main)
from __future__ import print_function, absolute_import from twisted.internet import reactor from twisted.logger import Logger, globalLogPublisher from cheeselib.logger import PrintingObserver from cheeselib.server.rpc import CheeseRPCServerFactory, CheeseRPCServer from cheeselib.server.storage.mongo import MongoDAO SERVER_PORT = 18080 log = Logger() globalLogPublisher.addObserver(PrintingObserver()) dao = MongoDAO() rpc_server = CheeseRPCServer(dao).getStreamFactory(CheeseRPCServerFactory) reactor.listenTCP(SERVER_PORT, rpc_server) log.info("Starting server on port %d..." % SERVER_PORT) reactor.run()
def startService(self): globalLogPublisher.addObserver(_count_errors) return Service.startService(self)
""" # The '-c' parameters is currently ignored optParameters = [] optFlags = [['help', 'h', 'Display this help and exit.']] @provider(ILogObserver) def importFailureObserver(event): if 'failure' in event and event['failure'].type is ImportError: log.err("ERROR: %s. Please run `pip install -U -r requirements.txt` " "from Cowrie's install directory and virtualenv to install " "the new dependency" % event['failure'].value.message) globalLogPublisher.addObserver(importFailureObserver) @implementer(IServiceMaker, IPlugin) class CowrieServiceMaker(object): tapname = "cowrie" description = "She sells sea shells by the sea shore." options = Options dbloggers = None output_plugins = None def makeService(self, options): """ Construct a TCPServer from a factory defined in Cowrie. """
def main(config_path=None): """ main. @type config_path: String @param config_path: optional alternative path to a config file """ log.debug("Welcome to the jungle!") try: cfg = load_config(config_path) if config_path else load_config() except ConfigObjError, e: log.failure(str(e)) site_plugins = load_site_plugins(cfg['sites']) db_plugin = load_database_plugin(cfg['database']) plugins = {'sites': site_plugins, 'database': db_plugin} import_data(plugins) if __name__ == '__main__': log_filter = LogLevelFilterPredicate(LogLevel.info) all_abserver = textFileLogObserver(open("spider.log", 'w')) filtered_observer = FilteringLogObserver(textFileLogObserver(sys.stdout), [log_filter]) globalLogPublisher.addObserver(all_abserver) globalLogPublisher.addObserver(filtered_observer) sys.exit(main())
def start(cls): globalLogPublisher.addObserver(getTextFileObserver()) cls.started = True
def main(): parser = argparse.ArgumentParser() # Network options group_network_container = parser.add_argument_group(title="Network options") group_network = group_network_container.add_mutually_exclusive_group(required=True) group_network.add_argument("--mainnet", action="store_true", default=False, help="Use MainNet") group_network.add_argument("--testnet", action="store_true", default=False, help="Use TestNet") group_network.add_argument("--privnet", action="store_true", default=False, help="Use PrivNet") group_network.add_argument("--coznet", action="store_true", default=False, help="Use CozNet") group_network.add_argument("--config", action="store", help="Use a specific config file") # Ports for RPC and REST api group_modes = parser.add_argument_group(title="Mode(s)") group_modes.add_argument("--port-rpc", type=int, help="port to use for the json-rpc api (eg. 10332)") group_modes.add_argument("--port-rest", type=int, help="port to use for the rest api (eg. 80)") # Advanced logging setup group_logging = parser.add_argument_group(title="Logging options") group_logging.add_argument("--logfile", action="store", type=str, help="Logfile") group_logging.add_argument("--syslog", action="store_true", help="Log to syslog instead of to log file ('user' is the default facility)") group_logging.add_argument("--syslog-local", action="store", type=int, choices=range(0, 7), metavar="[0-7]", help="Log to a local syslog facility instead of 'user'. Value must be between 0 and 7 (e.g. 0 for 'local0').") group_logging.add_argument("--disable-stderr", action="store_true", help="Disable stderr logger") # Where to store stuff parser.add_argument("--datadir", action="store", help="Absolute path to use for database directories") # peers parser.add_argument("--maxpeers", action="store", default=5, help="Max peers to use for P2P Joining") # host parser.add_argument("--host", action="store", type=str, help="Hostname ( for example 127.0.0.1)", default="0.0.0.0") # Now parse args = parser.parse_args() # print(args) if not args.port_rpc and not args.port_rest: print("Error: specify at least one of --port-rpc / --port-rest") parser.print_help() return if args.port_rpc == args.port_rest: print("Error: --port-rpc and --port-rest cannot be the same") parser.print_help() return if args.logfile and (args.syslog or args.syslog_local): print("Error: Cannot only use logfile or syslog at once") parser.print_help() return # Setup depending on command line arguments. By default, the testnet settings are already loaded. if args.config: settings.setup(args.config) elif args.mainnet: settings.setup_mainnet() elif args.testnet: settings.setup_testnet() elif args.privnet: settings.setup_privnet() elif args.coznet: settings.setup_coznet() if args.datadir: settings.set_data_dir(args.datadir) if args.maxpeers: settings.set_max_peers(args.maxpeers) if args.syslog or args.syslog_local is not None: # Setup the syslog facility if args.syslog_local is not None: print("Logging to syslog local%s facility" % args.syslog_local) syslog_facility = SysLogHandler.LOG_LOCAL0 + args.syslog_local else: print("Logging to syslog user facility") syslog_facility = SysLogHandler.LOG_USER # Setup logzero to only use the syslog handler logzero.syslog(facility=syslog_facility) else: # Setup file logging if args.logfile: logfile = os.path.abspath(args.logfile) if args.disable_stderr: print("Logging to logfile: %s" % logfile) else: print("Logging to stderr and logfile: %s" % logfile) logzero.logfile(logfile, maxBytes=LOGFILE_MAX_BYTES, backupCount=LOGFILE_BACKUP_COUNT, disableStderrLogger=args.disable_stderr) else: print("Logging to stdout and stderr") # Disable logging smart contract events settings.set_log_smart_contract_events(False) # Write a PID file to easily quit the service write_pid_file() # Setup Twisted and Klein logging to use the logzero setup observer = STDLibLogObserver(name=logzero.LOGZERO_DEFAULT_LOGGER) globalLogPublisher.addObserver(observer) # Instantiate the blockchain and subscribe to notifications blockchain = LevelDBBlockchain(settings.chain_leveldb_path) Blockchain.RegisterBlockchain(blockchain) dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks) dbloop.start(.1) # Setup twisted reactor, NodeLeader and start the NotificationDB reactor.suggestThreadPoolSize(15) NodeLeader.Instance().Start() NotificationDB.instance().start() # Start a thread with custom code d = threading.Thread(target=custom_background_code) d.setDaemon(True) # daemonizing the thread will kill it when the main thread is quit d.start() if args.port_rpc: logger.info("Starting json-rpc api server on http://%s:%s" % (args.host, args.port_rpc)) api_server_rpc = JsonRpcApi(args.port_rpc) # endpoint_rpc = "tcp:port={0}:interface={1}".format(args.port_rpc, args.host) # endpoints.serverFromString(reactor, endpoint_rpc).listen(Site(api_server_rpc.app.resource())) # reactor.listenTCP(int(args.port_rpc), server.Site(api_server_rpc)) api_server_rpc.app.run(args.host, args.port_rpc) if args.port_rest: logger.info("Starting REST api server on http://%s:%s" % (args.host, args.port_rest)) api_server_rest = RestApi() # endpoint_rest = "tcp:port={0}:interface={1}".format(args.port_rest, args.host) # endpoints.serverFromString(reactor, endpoint_rest).listen(Site(api_server_rest.app.resource())) api_server_rest.app.run(args.host, args.port_rest) reactor.run() # After the reactor is stopped, gracefully shutdown the database. logger.info("Closing databases...") NotificationDB.close() Blockchain.Default().Dispose() NodeLeader.Instance().Shutdown()
import io from twisted.logger import jsonFileLogObserver, globalLogPublisher from ad_hoc import AdHoc globalLogPublisher.addObserver(jsonFileLogObserver(io.open("log.json", "a"))) AdHoc(3, 4).logMessage()
def __enter__(self): set_global_log_level(self.desired_level) globalLogPublisher.addObserver(self.logs.append) return self
yield task.deferLater(nats_protocol.reactor, 1, reactor.stop) def main(reactor): host = "demo.nats.io" port = 4222 # TODO: make a NatsClient that does this, choosing the proper endpoint point = TCP4ClientEndpoint(reactor, host, port) nats_protocol = txnats.io.NatsProtocol(verbose=False, on_msg=my_on_msg, on_connect=somePubSubbing) # Because NatsProtocol implements the Protocol interface, Twisted's # connectProtocol knows how to connected to the endpoint. connecting = connectProtocol(point, nats_protocol) # Log if there is an error making the connection. connecting.addErrback(lambda np: log.info("{p}", p=np)) # Log what is returned by the connectProtocol. connecting.addCallback(lambda np: log.info("{p}", p=np)) return connecting if __name__ == '__main__': globalLogPublisher.addObserver(simpleObserver) main(reactor) reactor.run()
def run(): """ Entry point into (native) worker processes. This wires up stuff such that a worker instance is talking WAMP-over-stdio to the node controller. """ import os import sys import platform import signal # Ignore SIGINT so we get consistent behavior on control-C versus # sending SIGINT to the controller process. When the controller is # shutting down, it sends TERM to all its children but ctrl-C # handling will send a SIGINT to all the processes in the group # (so then the controller sends a TERM but the child already or # will very shortly get a SIGINT as well). Twisted installs signal # handlers, but not for SIGINT if there's already a custom one # present. def ignore(sig, frame): log.debug("Ignoring SIGINT in worker.") signal.signal(signal.SIGINT, ignore) # create the top-level parser # import argparse parser = argparse.ArgumentParser() parser.add_argument('--reactor', default=None, choices=['select', 'poll', 'epoll', 'kqueue', 'iocp'], help='Explicit Twisted reactor selection (optional).') parser.add_argument('--loglevel', default="info", choices=['none', 'error', 'warn', 'info', 'debug', 'trace'], help='Initial log level.') parser.add_argument('-c', '--cbdir', type=str, help="Crossbar.io node directory (required).") parser.add_argument('-n', '--node', type=str, help='Crossbar.io node ID (required).') parser.add_argument('-w', '--worker', type=str, help='Crossbar.io worker ID (required).') parser.add_argument('-r', '--realm', type=str, help='Crossbar.io node (management) realm (required).') parser.add_argument('-t', '--type', choices=['router', 'container'], help='Worker type (required).') parser.add_argument('--title', type=str, default=None, help='Worker process title to set (optional).') options = parser.parse_args() # make sure logging to something else than stdio is setup _first_ # from crossbar._logging import make_JSON_observer, cb_logging_aware, _stderr from crossbar._logging import make_logger, start_logging, set_global_log_level from twisted.logger import globalLogPublisher as log_publisher # Set the global log level set_global_log_level(options.loglevel) log = make_logger() # Print a magic phrase that tells the capturing logger that it supports # Crossbar's rich logging print(cb_logging_aware, file=_stderr) _stderr.flush() flo = make_JSON_observer(_stderr) log_publisher.addObserver(flo) start_logging() try: import setproctitle except ImportError: log.debug("Could not set worker process title (setproctitle not installed)") else: # set process title if requested to # if options.title: setproctitle.setproctitle(options.title) else: WORKER_TYPE_TO_TITLE = { 'router': 'crossbar-worker [router]', 'container': 'crossbar-worker [container]' } setproctitle.setproctitle(WORKER_TYPE_TO_TITLE[options.type].strip()) # we use an Autobahn utility to import the "best" available Twisted reactor # from autobahn.twisted.choosereactor import install_reactor reactor = install_reactor(options.reactor) from twisted.python.reflect import qual log.info("Worker running under {python}-{reactor}", python=platform.python_implementation(), reactor=qual(reactor.__class__).split('.')[-1]) options.cbdir = os.path.abspath(options.cbdir) os.chdir(options.cbdir) # log.msg("Starting from node directory {}".format(options.cbdir)) from crossbar.worker.router import RouterWorkerSession from crossbar.worker.container import ContainerWorkerSession WORKER_TYPE_TO_CLASS = { 'router': RouterWorkerSession, 'container': ContainerWorkerSession } from autobahn.twisted.websocket import WampWebSocketServerProtocol class WorkerServerProtocol(WampWebSocketServerProtocol): def connectionLost(self, reason): try: # this log message is unlikely to reach the controller (unless # only stdin/stdout pipes were lost, but not stderr) log.warn("Connection to node controller lost.") WampWebSocketServerProtocol.connectionLost(self, reason) except: pass finally: # losing the connection to the node controller is fatal: # stop the reactor and exit with error log.info("No more controller connection; shutting down.") reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) try: reactor.stop() except ReactorNotRunning: pass try: # create a WAMP application session factory # from autobahn.twisted.wamp import ApplicationSessionFactory from autobahn.wamp.types import ComponentConfig session_config = ComponentConfig(realm=options.realm, extra=options) session_factory = ApplicationSessionFactory(session_config) session_factory.session = WORKER_TYPE_TO_CLASS[options.type] # create a WAMP-over-WebSocket transport server factory # from autobahn.twisted.websocket import WampWebSocketServerFactory transport_factory = WampWebSocketServerFactory(session_factory, "ws://localhost", debug=False, debug_wamp=False) transport_factory.protocol = WorkerServerProtocol transport_factory.setProtocolOptions(failByDrop=False) # create a protocol instance and wire up to stdio # from twisted.python.runtime import platform as _platform from twisted.internet import stdio proto = transport_factory.buildProtocol(None) if _platform.isWindows(): stdio.StandardIO(proto) else: stdio.StandardIO(proto, stdout=3) # now start reactor loop # if False: log.info("vmprof enabled.") import os import vmprof PROFILE_FILE = 'vmprof_{}.dat'.format(os.getpid()) outfd = os.open(PROFILE_FILE, os.O_RDWR | os.O_CREAT | os.O_TRUNC) vmprof.enable(outfd, period=0.01) log.info("Entering event loop...") reactor.run() vmprof.disable() else: log.debug("Entering event loop...") reactor.run() except Exception as e: log.info("Unhandled exception: {}".format(e)) if reactor.running: reactor.addSystemEventTrigger('after', 'shutdown', os._exit, 1) reactor.stop() else: sys.exit(1)
def initCommandConfig(): if CMD_STATUS['mode'] == 'foreground': globalLogPublisher.addObserver(logObserver) elif CMD_STATUS['mode'] == 'background': ignoreKeyboardInterrupt()
optParameters = [] optFlags = [ ['help', 'h', 'Display this help and exit.'] ] @provider(ILogObserver) def importFailureObserver(event): if 'failure' in event and event['failure'].type is ImportError: log.err("ERROR: %s. Please run `pip install -U -r requirements.txt` " "from Cowrie's install directory and virtualenv to install " "the new dependency" % event['failure'].value.message) globalLogPublisher.addObserver(importFailureObserver) @implementer(IServiceMaker, IPlugin) class CowrieServiceMaker(object): tapname = "cowrie" description = "She sells sea shells by the sea shore." options = Options output_plugins = None def makeService(self, options): """ Construct a TCPServer from a factory defined in Cowrie. """ if options["help"] is True:
import os.path from flask import Flask, render_template from twisted.logger import globalLogPublisher from hendrix.deploy.base import HendrixDeploy from hendrix.experience import hey_joe from nucypher.characters.base import Character from nucypher.characters.lawful import Ursula from nucypher.config.constants import GLOBAL_DOMAIN from nucypher.network.middleware import RestMiddleware from nucypher.network.nodes import FleetStateTracker from nucypher.utilities.logging import SimpleObserver websocket_service = hey_joe.WebSocketService("127.0.0.1", 9000) globalLogPublisher.addObserver(SimpleObserver()) known_node = Ursula.from_seed_and_stake_info(seed_uri=sys.argv[1], federated_only=True, minimum_stake=0) rest_app = Flask("fleet-monitor", root_path=os.path.dirname(__file__)) class MonitoringTracker(FleetStateTracker): def record_fleet_state(self, *args, **kwargs): new_state_or_none = super(MonitoringTracker, self).record_fleet_state(*args, **kwargs) if new_state_or_none: checksum, new_state = new_state_or_none hey_joe.send({checksum: self.abridged_state_details(new_state)},
""" Start a Nats Protocol and connect it to a NATS endpoint over TCP4 which subscribes to a subject. """ log.info("Start client.") point = TCP4ClientEndpoint(reactor, host, port) nats_protocol = txnats.io.NatsProtocol( verbose=False, on_connect=lambda np: np.sub("happy", 6, on_msg=on_happy_msg)) # Because NatsProtocol implements the Protocol interface, Twisted's # connectProtocol knows how to connected to the endpoint. connecting = connectProtocol(point, nats_protocol) # Log if there is an error making the connection. connecting.addErrback(lambda np: log.info("{p}", p=np)) # Log what is returned by the connectProtocol. connecting.addCallback(lambda np: log.info("{p}", p=np)) return connecting def main(reactor): host = "demo.nats.io" port = 4222 create_client(reactor, host, port) if __name__ == '__main__': globalLogPublisher.addObserver(simpleObserver) main(reactor) reactor.run()
# Start the bot heufybot = HeufyBot(options.config) # Determine the logging level logFilter = LogLevelFilterPredicate() try: logFilter.setLogLevelForNamespace("heufybot", LogLevel.levelWithName(options.loglevel.lower())) invalidLogLevel = False except InvalidLogLevelError: logFilter.setLogLevelForNamespace("heufybot", LogLevel.info) invalidLogLevel = True # Set up logging logFile = DailyLogFile("heufybot.log", "") fileObserver = FileLogObserver(logFile, logFormat) fileFilterObserver = FilteringLogObserver(fileObserver, (logFilter,)) consoleFilterObserver = FilteringLogObserver(consoleLogObserver, (logFilter,)) heufybot.log = Logger("heufybot") globalLogPublisher.addObserver(fileFilterObserver) globalLogPublisher.addObserver(consoleFilterObserver) heufybot.log.info("Starting bot...") # Yell at the user if they specified an invalid log level if invalidLogLevel: heufybot.log.warn("Picked up invalid log level {invalidLevel}, level has been set to INFO instead.", invalidLevel=options.loglevel.lower()) signal(SIGINT, lambda signal, stack: heufybot.shutdown()) heufybot.startup()
def alicia_encrypt(data_fields): POLICY_FILENAME = "policy-metadata.json" globalLogPublisher.addObserver(SimpleObserver()) TEMP_ALICE_DIR = "alicia-files".format(os.path.dirname(os.path.abspath(__file__))) SEEDNODE_URL = '40.122.164.26:9151' passphrase = "TEST_ALICIA_INSECURE_DEVELOPMENT_PASSWORD" try: alice_config_file = os.path.join(TEMP_ALICE_DIR, "alice.config") new_alice_config = AliceConfiguration.from_configuration_file( filepath=alice_config_file, network_middleware=RestMiddleware(), start_learning_now=False, save_metadata=False, ) alicia = new_alice_config(passphrase=passphrase) except: shutil.rmtree(TEMP_ALICE_DIR, ignore_errors=True) ursula = Ursula.from_seed_and_stake_info(seed_uri=SEEDNODE_URL, federated_only=True, minimum_stake=0) alice_config = AliceConfiguration( config_root=os.path.join(TEMP_ALICE_DIR), is_me=True, known_nodes={ursula}, start_learning_now=False, federated_only=True, learn_on_same_thread=True, ) alice_config.initialize(password=passphrase) alice_config.keyring.unlock(password=passphrase) alicia = alice_config.produce() alice_config_file = alice_config.to_configuration_file() alicia.start_learning_loop(now=True) label = "doctor" label = label.encode() policy_pubkey = alicia.get_policy_pubkey_from_label(label) print("The policy public key for " "label '{}' is {}".format(label.decode("utf-8"), policy_pubkey.to_bytes().hex())) import data_ipfs res = data_ipfs.encrypt_patient_data(policy_pubkey , data_fields, label=label,save_as_file=True) print(res) from doctor_keys import get_doctor_pubkeys doctor_pubkeys = get_doctor_pubkeys() powers_and_material = { DecryptingPower: doctor_pubkeys['enc'], SigningPower: doctor_pubkeys['sig'] } doctor_strange = Bob.from_public_keys(powers_and_material=powers_and_material, federated_only=True) policy_end_datetime = maya.now() + datetime.timedelta(days=5) m, n = 2, 3 print("Creating access policy for the Doctor...") policy = alicia.grant(bob=doctor_strange, label=label, m=m, n=n, expiration=policy_end_datetime) print("Done!") policy_info = { "policy_pubkey": policy.public_key.to_bytes().hex(), "alice_sig_pubkey": bytes(alicia.stamp).hex(), "label": label.decode("utf-8"), } filename = POLICY_FILENAME with open(filename, 'w') as f: json.dump(policy_info, f) return res
def start(self): if self._filename: self._output = io.open(self._filename, "a", encoding="utf-8") if self.firehose: self.firehose.start() globalLogPublisher.addObserver(self)
from __future__ import absolute_import, division, print_function import sys from gaspocket.bot import run from twisted.internet.task import react from twisted.logger import globalLogPublisher, textFileLogObserver globalLogPublisher.addObserver(textFileLogObserver(sys.stdout)) if __name__ == '__main__': react(run, [])
import io from ad_hoc import AdHoc from twisted.logger import globalLogPublisher, jsonFileLogObserver globalLogPublisher.addObserver(jsonFileLogObserver(open("log.json", "a"))) AdHoc(3, 4).logMessage()