def process_nmap_commands(loggerName):
""" Main function. Here we set up the environment, factory, interface, and port """
global nmapCommandsFile
global nmapCommand
global port
global mlog
global verboseLevel
global clientTimeout
observer = log.PythonLoggingObserver(loggerName)
observer.start()
# Create the factory
factory = Factory()
factory.protocol = NmapServerProtocol
# Create the time based print
loop = task.LoopingCall(show_info)
loop.start(5.0) # call every second
# Create the time based file read
loop2 = task.LoopingCall(read_file_and_fill_nmap_variable)
loop2.start(30.0) # call every second
# To mark idle clients as hold
loop3 = task.LoopingCall(timeout_idle_clients)
loop3.start(clientTimeout) # call every second
# Create the reactor
reactor.listenSSL(port, factory, ServerContextFactory(), interface=interface)
reactor.run()
def main(stdscr):
parser = argparse.ArgumentParser()
parser.add_argument("user", type=str)
parser.add_argument("-v", "--verbose", action="count")
args = parser.parse_args()
user = args.user
server_name = origin_from_ucid(user)
## Set up logging ##
root_logger = logging.getLogger()
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
)
if not os.path.exists("logs"):
os.makedirs("logs")
fh = logging.FileHandler("logs/%s" % user)
fh.setFormatter(formatter)
root_logger.addHandler(fh)
root_logger.setLevel(logging.DEBUG)
# Hack: The only way to get it to stop logging to sys.stderr :(
log.theLogPublisher.observers = []
observer = log.PythonLoggingObserver()
observer.start()
## Set up synapse server
curses_stdio = cursesio.CursesStdIO(stdscr)
input_output = InputOutput(curses_stdio, user)
curses_stdio.set_callback(input_output)
app_hs = SynapseHomeServer(server_name, db_name="dbs/%s" % user)
replication = app_hs.get_replication_layer()
hs = HomeServer(server_name, replication, curses_stdio)
input_output.set_home_server(hs)
## Add input_output logger
io_logger = IOLoggerHandler(input_output)
io_logger.setFormatter(formatter)
root_logger.addHandler(io_logger)
## Start! ##
try:
port = int(server_name.split(":")[1])
except:
port = 12345
app_hs.get_http_server().start_listening(port)
reactor.addReader(curses_stdio)
reactor.run()
def configure_logging():
log_format = '%(asctime)-15s %(levelname)s %(message)s'
logging.basicConfig(format=log_format)
logger = logging.getLogger('port_agent')
logger.setLevel(logging.INFO)
observer = log.PythonLoggingObserver('port_agent')
observer.start()
def create_loggers(level=logging.INFO):
# Pipe Twisted's loggers into python logging package
log_observer = twisted_log.PythonLoggingObserver()
log_observer.start()
# Python logging setup
# TODO: Make this configurable (https://github.com/bbc/ebu-tt-live-toolkit/issues/15)
logging.basicConfig(level=level, format=log_format)
def get_application(*args):
config_file_path = os.environ.get('INTRANET2_CONFIG', '')
assert config_file_path, u'Config file path must be given in INTRANET2_CONFIG environment variable'
observer = log.PythonLoggingObserver()
observer.start()
config = ConfigParser.ConfigParser()
config.read(config_file_path)
from intranet3 import main
paster.setup_logging(config_file_path)
settings = paster.get_appsettings(config_file_path)
app = main(None, **settings)
from intranet3 import cron
if not settings.get('CRON_DISABLE'):
cron.run_cron_tasks()
if config.getboolean('server:main', 'proxy_fix'):
from werkzeug.contrib.fixers import ProxyFix
app = ProxyFix(app)
if config.getboolean('server:main', 'scheme_fix'):
app = HttpsSchemeFixer(app)
return app
def startWebServer(options, config):
# Extract properties from config
log_dir = str(config["configuration"]["log_dir"])
content = str(config["configuration"]["content"])
endpoint = str(config["configuration"]["endpoint"])
host = str(config["configuration"]["host"])
port = int(config["configuration"]["port"])
# Setup logging
logFileName = log_dir + os.sep + "launcherLog.log"
formatting = '%(asctime)s:%(levelname)s:%(name)s:%(message)s'
logging.basicConfig(level=logging.DEBUG, filename=logFileName, filemode='w', format=formatting)
observer = log.PythonLoggingObserver()
observer.start()
if options.debug:
console = logging.StreamHandler(sys.stdout)
console.setLevel(logging.INFO)
formatter = logging.Formatter(formatting)
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
# Initialize web resource
web_resource = File(content) if (len(content) > 0) else resource.Resource()
# Attach launcher
web_resource.putChild(endpoint, LauncherResource(options, config))
site = server.Site(web_resource)
reactor.listenTCP(port, site, interface=host)
reactor.run()
def run(self):
if not self.factory:
self._init_factory()
if self.config and self.factory and self.reactor:
if self.config.ssl:
from twisted.internet import ssl
self.reactor.connectSSL(self.config.server,
self.config.port, self.factory,
ssl.ClientContextFactory())
else:
self.reactor.connectTCP(self.config.server, self.config.port,
self.factory)
# Activate webservice
if self.webresource is not None:
from twisted.web.server import Site
site_factory = Site(self.webresource)
self.reactor.listenTCP(self.config.webserver_port,
site_factory)
# Add sentry error handler
if hasattr(self.config, 'sentry_dsn'):
from twisted.python import log
from raven.handlers.logging import SentryHandler
self.sentry = SentryHandler(self.config.sentry_dsn)
logging.getLogger('twisted').addHandler(self.sentry)
self.observer = log.PythonLoggingObserver()
self.observer.start()
self.reactor.run()
def init_logging(level):
logging.basicConfig(stream=sys.stdout,
format=LogFormat,
datefmt=DateFormat,
level=level)
log.PythonLoggingObserver().start()
def start(logfile=None, application_name="ooniprobe"):
daily_logfile = None
if not logfile:
logfile = config.basic.logfile
log_folder = os.path.dirname(logfile)
log_filename = os.path.basename(logfile)
daily_logfile = DailyLogFile(log_filename, log_folder)
txlog.msg(
"Starting %s on %s (%s UTC)" %
(application_name, otime.prettyDateNow(), otime.utcPrettyDateNow()))
logging.basicConfig()
python_logging = txlog.PythonLoggingObserver(application_name)
if config.advanced.debug:
python_logging.logger.setLevel(logging.DEBUG)
else:
python_logging.logger.setLevel(logging.INFO)
txlog.startLoggingWithObserver(python_logging.emit)
txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)
def setup_logging(config: SydentConfig) -> None:
"""
Setup logging using the options specified in the config
:param config: the configuration to use
"""
log_path = config.general.log_path
log_level = config.general.log_level
log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s"
formatter = logging.Formatter(log_format)
handler: logging.Handler
if log_path != "":
handler = logging.handlers.TimedRotatingFileHandler(
log_path, when="midnight", backupCount=365
)
handler.setFormatter(formatter)
else:
handler = logging.StreamHandler()
handler.setFormatter(formatter)
rootLogger = logging.getLogger("")
rootLogger.setLevel(log_level)
rootLogger.addHandler(handler)
observer = log.PythonLoggingObserver()
observer.start()
def __init__(self, name, maxkbytes=1024, count=5, console=True):
'''
Using this class you can add logging to your plug-in.
It provides a generic way of storing logging information.
@param name: the name of the logfile
@param maxkbytes: the maximum logfile size in kilobytes
@param count: the maximum number of logfiles to keep for rotation
@param console: specifies whether or not to log to the console, this defaults to "True"
'''
# Start Twisted python log observer
observer = twisted_log.PythonLoggingObserver()
observer.start()
# Get logpath
config = Config(config_file)
# Regular Python logging module
self.logger = logging.getLogger()
log_handler = logging.handlers.RotatingFileHandler(
filename=os.path.join(config.general.logpath, "%s.log" % name),
maxBytes=config.general.logsize * 1024,
backupCount=config.general.logcount)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
if config.general.logconsole:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(formatter)
log_handler.setFormatter(formatter)
self.logger.addHandler(log_handler)
self.logger.addHandler(console_handler)
def logging_setup(filename, sentry_dsn):
'''Sets up the logging system to output to stdout and filename,
if filename is not None'''
LOGGING_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
if not os.environ.get('JUNEBUG_DISABLE_LOGGING'):
# Send Twisted Logs to python logger
if sentry_dsn:
PythonExceptionFilteringLoggingObserver().start()
else:
log.PythonLoggingObserver().start()
# Set up stdout logger
logging.basicConfig(level=logging.INFO,
format=LOGGING_FORMAT,
stream=sys.stdout)
# Set up file logger
if filename:
handler = logging.handlers.RotatingFileHandler(filename,
maxBytes=1024 * 1024,
backupCount=5)
handler.setFormatter(logging.Formatter(LOGGING_FORMAT))
logging.getLogger().addHandler(handler)
def main(argv):
from twisted.python import log
from twisted.web.server import Site
from twisted.web.static import File
from twisted.internet import reactor
from twisted.python import log
observer = log.PythonLoggingObserver('twisted')
log.startLoggingWithObserver(observer.emit, setStdout=False)
static_dir = os.path.abspath('.')
logging.info("registering static folder %r on /" % static_dir)
root = File(static_dir)
wr = TwistedWebResource(soap11_application)
logging.info("registering %r on /%s" % (wr, url))
root.putChild(url, wr)
site = Site(root)
if port[0] == 0:
port[0] = get_open_port()
reactor.listenTCP(port[0], site)
logging.info("listening on: %s:%d" % (host,port))
return reactor.run()
def setup_logging(cfg):
log_format = ("%(asctime)s - %(name)s - %(lineno)d - %(levelname)s"
" - %(message)s")
formatter = logging.Formatter(log_format)
logPath = cfg.get('general', "log.path")
if logPath != '':
handler = logging.handlers.TimedRotatingFileHandler(logPath,
when='midnight',
backupCount=365)
handler.setFormatter(formatter)
def sighup(signum, stack):
logger.info("Closing log file due to SIGHUP")
handler.doRollover()
logger.info("Opened new log file due to SIGHUP")
else:
handler = logging.StreamHandler()
handler.setFormatter(formatter)
rootLogger = logging.getLogger('')
rootLogger.setLevel(cfg.get('general', 'log.level'))
rootLogger.addHandler(handler)
observer = log.PythonLoggingObserver()
observer.start()
def setup_logging(options):
default = pkg_resources.resource_filename(tron.__name__, 'logging.conf')
logfile = options.log_conf or default
level = twist_level = None
if options.verbose > 0:
level = logging.INFO
twist_level = logging.WARNING
if options.verbose > 1:
level = logging.DEBUG
twist_level = logging.INFO
if options.verbose > 2:
twist_level = logging.DEBUG
tron_logger = logging.getLogger('tron')
twisted_logger = logging.getLogger('twisted')
logging.config.fileConfig(logfile)
if level is not None:
tron_logger.setLevel(level)
if twist_level is not None:
twisted_logger.setLevel(twist_level)
# Hookup twisted to standard logging
twisted_log.PythonLoggingObserver().start()
# Show stack traces for errors in twisted deferreds.
if options.debug:
defer.setDebugging(True)
def main(argv):
observer = log.PythonLoggingObserver('twisted')
log.startLoggingWithObserver(observer.emit, setStdout=False)
wsgi_application = WsgiApplication(soap_application)
return run_twisted([(wsgi_application, url)], port)
def start(self):
app_for_context = create_app()
dns_logging = DatabaseDNSLogging(app_for_context)
clients = [
DatabaseDNSResolver(app_for_context,
Provider().dns_manager(), dns_logging)
]
if self.__forwarding_enabled and len(self.__forwarders) > 0:
clients.append(client.Resolver(servers=self.__forwarders))
factory = DatabaseDNSFactory(clients=clients, verbose=2)
factory.app = app_for_context
factory.logging = dns_logging
factory.restrictions = Provider().dns_restrictions()
factory.csv_location = self.__csv_location
observer = log.PythonLoggingObserver()
observer.start()
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
reactor.listenUDP(self.__port,
DNSDatagramProtocol(factory),
interface=self.__host)
reactor.listenTCP(self.__port, factory, interface=self.__host)
reactor.run()
def __init__(self,rootUri="http://localhost",environmentManager=None,envId=None,nodeId=None):
DefaultRestHandler.__init__(self,rootUri)
self.logger=log.PythonLoggingObserver("dobozweb.core.server.rest.driverStatusHandler")
self.environmentManager=environmentManager
self.envId=envId
self.nodeId=nodeId
self.valid_contentTypes.append("application/pollapli.driver.status+json")
def process_nmap_commands(logger_name):
""" Main function. Here we set up the environment, factory and port """
global nmap_commands_file
global nmap_command
global port
global mlog
global verbose_level
global client_timeout
observer = log.PythonLoggingObserver(logger_name)
observer.start()
# Create the factory
factory = Factory()
factory.protocol = NmapServerProtocol
# Create the time based print
loop = task.LoopingCall(show_info)
loop.start(5)
# Create the time based file read
loop2 = task.LoopingCall(read_file_and_fill_nmap_variable)
loop2.start(1)
# To mark idel clients as hold
loop3 = task.LoopingCall(timeout_idle_clients)
loop3.start(client_timeout) # call every second
if not sql_file == "":
loop4 = task.LoopingCall(sql_import_loop)
loop4.start(5)
# Create the reactor
reactor.listenSSL(port, factory, ServerContextFactory())
reactor.run()
def MainRun(isdaemon, id):
random.seed(time.time())
logging.getLogger().setLevel(config.instance.log_level)
handler = TimedRotatingFileHandler(filename=config.instance.log_file,
when='D',
interval=1)
handler.setLevel(config.instance.log_level)
formatter = logging.Formatter(config.instance.log_format)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
log.PythonLoggingObserver().start()
if not isdaemon:
handler = logging.StreamHandler()
handler.setLevel(config.instance.log_level)
formatter = logging.Formatter(config.instance.log_format)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
#建立socket监听
clientfactory.instance.start(config.instance.server_ip,
config.instance.server_port,
config.instance.max_client)
redishelper.instance.start()
logging.info(u"登录网关服务器启动成功!服务器ID:%u", id)
def redirect_twisted_logging(logger_name=None, log_level=None):
# type: (Optional[str], Optional[Union[int, str]]) -> None
"""
Redirect Twisted's log messages to the Python logging module.
Twisted has its own logging system (several, actually). Calling this method redirects all Twisted log messages to
Python logging, by default as logger name "twisted", and for levels WARNING and up.
:param logger_name: Name of the Twisted logger, so you can retrieve and configure it with `getLogger(...)`
:param log_level: Log level for the messages to be forward. Default is WARNING
"""
if logger_name is None:
logger_name = "twisted"
if log_level is None:
log_level = logging.WARNING
# Redirect messages from Twisted's log system to Python logging
observer = twisted_log.PythonLoggingObserver(loggerName=logger_name)
observer.start()
# Adjust the log level, because debug and info messages are too verbose for most use cases
logging.getLogger(logger_name).setLevel(log_level)
# Twisted will by default print error messages to stderr until it has been instructed via `startLogging` to use a
# different log handler. We silence Twisted's logging here, since we already redirect to Python's logging system.
# There must be a better way to do this!
twisted_log.startLogging(open(os.devnull, "w"))
def init():
logging.basicConfig(stream=sys.stdout,
format=LogFormat,
datefmt=DateFormat,
level=logging.DEBUG)
twistedlog.PythonLoggingObserver().start()
def setupLogging():
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(module)s:%(lineno)d - %(message)s',
'%H:%M:%S')
# Setup stdout logging
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setLevel(logging.DEBUG)
stdout_handler.setFormatter(formatter)
# Setup buffer logging (for web display)
log_buffer = StringIO.StringIO()
buffer_handler = logging.StreamHandler(log_buffer)
buffer_handler.setLevel(logging.WARNING)
buffer_handler.setFormatter(formatter)
# Setup the root logger to use both handlers.
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger.addHandler(stdout_handler)
logger.addHandler(buffer_handler)
# Direct twisted logs into the standard Python logging.
observer = log.PythonLoggingObserver()
observer.start()
return buffer_handler, log_buffer
def verify_missing(zmq_endpoint):
"""Check that all messages go through AMQP and ZeroMQ."""
if tw_log is None:
raise click.exceptions.UsageError(
"You need to install Twisted to use this command."
)
# When the current RHEL has Twisted 15+, we can use twisted.logger
# tw_logger.globalLogPublisher.addObserver(
# tw_logger.STDLibLogObserver(name="verify_missing")
# )
# Send all the logs to stdlib's logging library
tw_log.PythonLoggingObserver(loggerName="verify_missing").start()
tw_log.startLogging(tw_log.NullFile())
zmq_endpoints = zmq_endpoint or config.conf["zmq_to_amqp"]["zmq_endpoints"]
if not zmq_endpoints:
raise click.exceptions.UsageError(
"No ZeroMQ endpoints defined, please provide one or more endpoints "
"using the --zmq-endpoint flag or by setting endpoints in the "
'"zmq_to_amqp" section of your configuration.'
)
try:
verify_missing_module.main(zmq_endpoints)
except zmq.error.ZMQError as e:
_log.exception(e)
except Exception:
_log.exception("An unexpected error occurred, please file a bug report")
def __init__(self):
logger.info("Starting Sydent server")
self.parse_config()
logPath = self.cfg.get('general', "log.path")
if logPath != '':
logging.basicConfig(level=logging.INFO, filename=logPath)
else:
logging.basicConfig(level=logging.INFO, filename=logPath)
self.pidfile = self.cfg.get('general', "pidfile.path");
observer = log.PythonLoggingObserver()
observer.start()
self.db = SqliteDatabase(self).db
self.server_name = self.cfg.get('general', 'server.name')
if self.server_name == '':
self.server_name = os.uname()[1]
logger.warn(("You had not specified a server name. I have guessed that this server is called '%s' "
+ " and saved this in the config file. If this is incorrect, you should edit server.name in "
+ "the config file.") % (self.server_name,))
self.cfg.set('general', 'server.name', self.server_name)
self.save_config()
self.validators = Validators()
self.validators.email = EmailValidator(self)
self.validators.msisdn = MsisdnValidator(self)
self.keyring = Keyring()
self.keyring.ed25519 = SydentEd25519(self).signing_key
self.keyring.ed25519.alg = 'ed25519'
self.servlets = Servlets()
self.servlets.emailRequestCode = EmailRequestCodeServlet(self)
self.servlets.emailValidate = EmailValidateCodeServlet(self)
self.servlets.msisdnRequestCode = MsisdnRequestCodeServlet(self)
self.servlets.msisdnValidate = MsisdnValidateCodeServlet(self)
self.servlets.lookup = LookupServlet(self)
self.servlets.bulk_lookup = BulkLookupServlet(self)
self.servlets.pubkey_ed25519 = Ed25519Servlet(self)
self.servlets.pubkeyIsValid = PubkeyIsValidServlet(self)
self.servlets.ephemeralPubkeyIsValid = EphemeralPubkeyIsValidServlet(self)
self.servlets.threepidBind = ThreePidBindServlet(self)
self.servlets.replicationPush = ReplicationPushServlet(self)
self.servlets.getValidated3pid = GetValidated3pidServlet(self)
self.servlets.storeInviteServlet = StoreInviteServlet(self)
self.servlets.blindlySignStuffServlet = BlindlySignStuffServlet(self)
self.threepidBinder = ThreepidBinder(self)
self.sslComponents = SslComponents(self)
self.clientApiHttpServer = ClientApiHttpServer(self)
self.replicationHttpsServer = ReplicationHttpsServer(self)
self.replicationHttpsClient = ReplicationHttpsClient(self)
self.pusher = Pusher(self)
def hook_twisted(levels=None, redirect_stdout=0):
_ensure_main_logger()
if levels:
set_levels(levels)
from twisted.python import log
plo = log.PythonLoggingObserver(TWISTED_CATEGORY)
log.startLoggingWithObserver(plo.emit, setStdout=redirect_stdout)
def setup(level=logging.DEBUG):
'''
Setup logging bridge from twisted to python
'''
observer = log.PythonLoggingObserver()
observer.start()
logging.basicConfig(level=logging.DEBUG)
def __init__(self, envPath):
self.logger = log.PythonLoggingObserver(
"dobozweb.core.components.environments.environmentManager")
self.path = EnvironmentManager.envPath
self.idCounter = EnvironmentManager.idCounter
self._signal_channel = "environment_manager"
self.signalHandler = SignalHander(self._signal_channel)
def start(self):
"""Starts the runnable, so that it regularly calls performDuty()"""
logging.debug("Started performing duties every %d seconds" % self.pauseBetweenDuties)
observer = log.PythonLoggingObserver()
observer.start()
lcDeferred = self.lc.start(self.pauseBetweenDuties)
lcDeferred.addErrback(self.errback)
def initialize(services, tns='spyne.examples.twisted.resource'):
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
observer = log.PythonLoggingObserver('twisted')
log.startLoggingWithObserver(observer.emit, setStdout=False)
return Application(services, 'spyne.examples.twisted.hello',
in_protocol=HttpRpc(), out_protocol=HttpRpc())
