def start(logfile=None, application_name="ooniprobe"):
daily_logfile = None
if not logfile:
logfile = config.basic.logfile
log_folder = os.path.dirname(logfile)
log_filename = os.path.basename(logfile)
daily_logfile = DailyLogFile(log_filename, log_folder)
txlog.msg(
"Starting %s on %s (%s UTC)" %
(application_name, otime.prettyDateNow(), otime.utcPrettyDateNow()))
logging.basicConfig()
python_logging = txlog.PythonLoggingObserver(application_name)
if config.advanced.debug:
python_logging.logger.setLevel(logging.DEBUG)
else:
python_logging.logger.setLevel(logging.INFO)
txlog.startLoggingWithObserver(python_logging.emit)
txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)
def createApplication():
appName = getAppConfig('app_name')
apiServerIp = getAppConfig('api_server.bind_address')
apiServerPort = int(getAppConfig('api_server.port'))
games = getAppConfig('games').split(",")
application = service.Application(name=appName)
# log.msg(orderServerOption.get('nodaemon'))
# log.msg(orderServerOption.keys())
# log.msg(orderServerOption.values())
# if orderServerOption.get('nodaemon') != 0:
logDir = os.path.realpath(os.path.dirname(getAppConfig('logdir')))
logName = "%s.log" % ( appName, )
logFile = logfile.DailyLogFile(logName, logDir)
#log.msg(logDir)
application.setComponent(log.ILogObserver, log.FileLogObserver(logFile).emit)
log.msg("load application: ", appName)
log.msg("load games:", games)
appSrv = service.IServiceCollection(application)
_apiService = apiService(bindAddress=apiServerIp, port=apiServerPort, games=games)
_apiService.setServiceParent(appSrv)
_consumerService = consumerService(games=games)
_consumerService.setServiceParent(appSrv)
return application
def set_log_file(logFile, setStdout=True, start=True):
"""Set up twisted log files in a standard way.
This borrows functionality from the twisted.application.app
AppLogger.start() method, which in turn uses the
twisted.scripts._twistd_unix.UnixAppLogger._getLogObserver method.
That method returns a FileLog that is hard-coded to set a log
rotation of 1 MB. While we're here anyway, overriding that as well
to remove the rotation size. We rotate logs with an external
logrotate command that sends SIGUSR1 to tell the broker to reopen
the log file.
"""
log_file = logfile.LogFile.fromFullPath(logFile, rotateLength=0)
observer = log.FileLogObserver(log_file).emit
try:
import signal
except ImportError:
pass
else:
# Override if signal is set to None or SIG_DFL (0)
if not signal.getsignal(signal.SIGUSR1):
def restartLog(signal, frame):
log_file.flush()
try:
log_file.close()
except:
pass
log_file._openFile()
signal.signal(signal.SIGUSR1, restartLog)
if start:
log.startLoggingWithObserver(observer, setStdout=setStdout)
return observer
def _run_core(self):
# Add an observer to trap all logged errors.
self.case.reactor = self._reactor
error_observer = _log_observer
full_log = StringIO()
full_observer = log.FileLogObserver(full_log)
spinner = self._make_spinner()
successful, unhandled = run_with_log_observers(
[error_observer.gotEvent, full_observer.emit],
self._blocking_run_deferred, spinner)
self.case.addDetail('twisted-log', text_content(full_log.getvalue()))
logged_errors = error_observer.flushErrors()
for logged_error in logged_errors:
successful = False
self._got_user_failure(logged_error, tb_label='logged-error')
if unhandled:
successful = False
for debug_info in unhandled:
f = debug_info.failResult
info = debug_info._getDebugTracebacks()
if info:
self.case.addDetail('unhandled-error-in-deferred-debug',
text_content(info))
self._got_user_failure(f, 'unhandled-error-in-deferred')
junk = spinner.clear_junk()
if junk:
successful = False
self._log_user_exception(UncleanReactorError(junk))
if successful:
self.result.addSuccess(self.case, details=self.case.getDetails())
def _setUp(self):
logs = StringIO()
full_observer = log.FileLogObserver(logs)
self.useFixture(_TwistedLogObservers([full_observer.emit]))
self.addDetail(
self.LOG_DETAIL_NAME,
Content(UTF8_TEXT, lambda: [logs.getvalue().encode("utf-8")]))
def build(self):
"""Main build function of the Kivy application."""
#
# Setup logging.
#
if not os.path.exists(gs.AUVSI_BASE_FOLDER):
os.makedirs(gs.AUVSI_BASE_FOLDER)
log.startLogging(stdout)
log.addObserver(
log.FileLogObserver(
file(os.path.join(gs.AUVSI_BASE_FOLDER, 'controller.log'),
'a+')).emit)
self.settings_cls = SettingsWithTabbedPanel
#
# Start up the local server.
#
self.connect_to_server()
#
# Load previously saved data
#
self.populateGalleryLists()
def test_emitPrefix(self):
"""
FileLogObserver.emit() will add a timestamp and system prefix to its
file output.
"""
output = StringIO()
flo = log.FileLogObserver(output)
events = []
def observer(event):
# Capture the event for reference and pass it along to flo
events.append(event)
flo.emit(event)
publisher = log.LogPublisher()
publisher.addObserver(observer)
publisher.msg("Hello!")
self.assertEqual(len(events), 1)
event = events[0]
result = output.getvalue()
prefix = "{time} [{system}] ".format(
time=flo.formatTime(event["time"]), system=event["system"],
)
self.assertTrue(
result.startswith(prefix),
"{0!r} does not start with {1!r}".format(result, prefix)
)
def main():
log_file = logfile.LogFile.fromFullPath('log/serverlog.log')
log.addObserver(log.FileLogObserver(log_file).emit)
print("===== PSO2Proxy vGIT %s =====" % config.proxy_ver)
time_string = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
print("[ServerStart] Trying to start server at %s" % time_string)
if myIp == "0.0.0.0":
print("==== ERROR 001 ====")
print("You have NOT configured the IP address for PSO2Proxy!")
print(
"Please edit cfg/pso2proxy.config.yml and change myIpAddr to your IP public IP address "
"(Not LAN address if you're on a LAN!) ")
print("After you fix this, please restart PSO2Proxy.")
sys.exit(0)
if bindIp == "0.0.0.0":
interface_ip = myIp
else:
interface_ip = bindIp
if not os.path.isfile("keys/myKey.pem"):
print("==== ERROR 002 ====")
print("You do NOT have your local RSA private key installed to 'keys/myKey.pem'!")
print("Please see README.md's section on RSA keys for more information.")
print("After you fix this, please restart PSO2Proxy.")
sys.exit(0)
if not os.path.isfile("keys/SEGAKey.pem"):
print("==== ERROR 003 ====")
print("You do NOT have a SEGA RSA public key installed to 'keys/SEGAKey.pem'!")
print("Please see README.md's section on RSA keys for more information.")
print("After you fix this, please restart PSO2Proxy.")
sys.exit(0)
for shipNum in range(0, 10): # PSO2 Checks all ships round robin, so sadly for max compatibility we have to open these no matter what ships are enabled...
ship_endpoint = endpoints.TCP4ServerEndpoint(reactor, 12099 + (100 * shipNum), interface=interface_ip)
ship_endpoint.listen(ShipAdvertiserFactory())
for shipNum in config.globalConfig.get_key('enabledShips'):
query_endpoint = endpoints.TCP4ServerEndpoint(reactor, 12000 + (100 * shipNum), interface=interface_ip)
query_endpoint.listen(BlockScraperFactory())
print("[ShipProxy] Bound port %i for ship %i query server!" % ((12000 + (100 * shipNum)), shipNum))
query_endpoint = endpoints.TCP4ServerEndpoint(reactor, 13000, interface=interface_ip)
query_endpoint.listen(BlockScraperFactory())
stdio.StandardIO(ServerConsole())
print("[ShipProxy] Loading plugins...")
import glob
for plug in glob.glob("plugins/*.py"):
plug = plug[:-3]
plug = plug.replace(os.sep, '.')
print("[ShipProxy] Importing %s..." % plug)
__import__(plug)
for f in plugin_manager.onStart:
f()
reactor.suggestThreadPoolSize(30)
reactor.run()
data.clients.dbManager.close_db()
for f in plugin_manager.onStop:
f()
def main():
log.startLogging(sys.stdout, setStdout=True)
logFile = logfile.LogFile.fromFullPath('twistedLog.log')
log.addObserver(log.FileLogObserver(logFile).emit)
s = Server()
s.run()
def setup_log(self, name):
try:
self.log_file = open(name, 'a')
self.log_observer = log.FileLogObserver(self.log_file)
log.startLoggingWithObserver(self.log_observer.emit)
except:
msg = "Error in setup_log:\n%s" % traceback.format_exc()
print msg
mail.error(msg)
def _setUpLogFile(self):
self._tearDownLogFile()
if self.logfile == "-":
logFile = sys.stdout
else:
logFile = util.openTestLog(filepath.FilePath(self.logfile))
self._logFileObject = logFile
self._logFileObserver = log.FileLogObserver(logFile)
log.startLoggingWithObserver(self._logFileObserver.emit, 0)
def _setUpLogFile(self):
self._tearDownLogFile()
if self.logfile == "-":
logFile = sys.stdout
else:
logFile = open(self.logfile, "a")
self._logFileObject = logFile
self._logFileObserver = log.FileLogObserver(logFile)
log.startLoggingWithObserver(self._logFileObserver.emit, 0)
def run(self,
serverLogFilename: str = 'twistedLog.log',
timeout: Optional[float] = None,
callOnTimer: Optional[Callable[[], None]] = None):
'''
Run the server. This method will not return
until the server is ended by an Exception like ^C.
Timeout is useful when actions need to be triggered
on a timer, with timout being a float representing
the seconds between calls.
If `callOnTimer` is set, it will be called every time
the timer triggers. Additionally, any game implementing
the `AbstractTimeGame` will have its `onTimer` method
be called.
'''
log.startLogging(sys.stdout, setStdout=True)
logFile = logfile.LogFile.fromFullPath(serverLogFilename)
log.addObserver(log.FileLogObserver(logFile).emit)
if callOnTimer is not None and not callable(callOnTimer):
raise TypeError(
f'Func specified, but it is not callable: "{type(callOnTimer)}"'
)
def loopCall() -> None:
if callOnTimer:
callOnTimer()
gDB = self.__requestProcessor.gameDatabase
gameIDs = gDB.getAllGameIDs()
for gID in gameIDs:
game = gDB.getGame(gID)
if game and isinstance(game, AbstractTimeGame):
result = game.onTimer()
if result is None:
continue
if isinstance(result, interactions.TimerResponse):
self.server.handleTimerResponse(result)
else:
raise TypeError(
f'Method onTimer did not return Optional[ResponseSuccess] but instead "{type(result)}" in game "{type(game)}"'
)
try:
if timeout:
loop = task.LoopingCall(loopCall)
loop.start(timeout)
# start listening for and handling connections
reactor.run() # pylint: disable=no-member
except KeyboardInterrupt:
print('KeyboardInterrupt')
finally:
self.logFile.close()
def _getLogObserver(self):
"""
Create a log observer to be added to the logging system before running
this application.
"""
if self._logfilename == '-' or not self._logfilename:
logFile = sys.stdout
else:
logFile = logfile.LogFile.fromFullPath(self._logfilename)
return log.FileLogObserver(logFile).emit
def setUp(self):
# F**k you Python.
reload(sys)
self._origEncoding = sys.getdefaultencoding()
sys.setdefaultencoding('ascii')
self.out = FakeFile()
self.lp = log.LogPublisher()
self.flo = log.FileLogObserver(self.out)
self.lp.addObserver(self.flo.emit)
def start(self, application):
logging.basicConfig()
python_logging = txlog.PythonLoggingObserver()
if config.debug:
python_logging.logger.setLevel(logging.DEBUG)
else:
python_logging.logger.setLevel(logging.INFO)
txlog.startLoggingWithObserver(python_logging.emit)
txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)
def on_load(self):
config = self.server.config.base
logfile = config.log_name
if config.rotate_daily:
create_file_path(logfile)
logging_file = DailyLogFile(logfile, '.')
else:
logging_file = open_create(logfile, 'a')
self.file_observer = log.FileLogObserver(logging_file)
log.addObserver(self.file_observer.emit)
log.msg('cuwo server started on %s' % time.strftime('%c'))
self.observer = log.startLogging(sys.stdout) # force twisted logging
def _getLogObserver(logfilename, sysLog, prefix, nodaemon):
"""
Create and return a suitable log observer for the given configuration.
The observer will go to syslog using the prefix C{prefix} if C{sysLog} is
true. Otherwise, it will go to the file named C{logfilename} or, if
C{nodaemon} is true and C{logfilename} is C{"-"}, to stdout.
@type logfilename: C{str}
@param logfilename: The name of the file to which to log, if other than the
default.
@type sysLog: C{bool}
@param sysLog: A flag indicating whether to use syslog instead of file
logging.
@type prefix: C{str}
@param prefix: If C{sysLog} is C{True}, the string prefix to use for syslog
messages.
@type nodaemon: C{bool}
@param nodaemon: A flag indicating the process will not be daemonizing.
@return: An object suitable to be passed to C{log.addObserver}.
"""
if sysLog:
observer = syslog.SyslogObserver(prefix).emit
else:
if logfilename == '-':
if not nodaemon:
print 'daemons cannot log to stdout'
os._exit(1)
logFile = sys.stdout
elif nodaemon and not logfilename:
logFile = sys.stdout
else:
logFile = logfile.LogFile.fromFullPath(logfilename or 'twistd.log')
try:
import signal
except ImportError:
pass
else:
def rotateLog(signal, frame):
from twisted.internet import reactor
reactor.callFromThread(logFile.rotate)
signal.signal(signal.SIGUSR1, rotateLog)
observer = log.FileLogObserver(logFile).emit
return observer
def main():
if len(sys.argv) < 2:
print("Usage: %s <config dir>" % sys.argv[0])
sys.exit(1)
transportobj = transport.Transport()
boss = pluginbase.PluginBoss(sys.argv[1], transportobj)
observer = log.FileLogObserver(sys.stdout)
observer.timeFormat = "%Y-%m-%d %H:%M:%S"
log.startLoggingWithObserver(observer.emit)
log.msg("Abbott starting up!")
boss.load_all_plugins()
reactor.run()
def updated_application_run(self):
"""Patch for twisted.application.app.ApplicationRunner.run().
We really want logging to start up before the app does. This has
needed to change in every upgrade from 8.1.0 -> 8.2.0 -> 10.2.0.
The difference for 8.2.0 is that self.logger.start() is exploded to
remove the requirement that the application exists and the
application created afterwards. In 10.2.0 the logger.start()
implementation changed. The logic structure has been modified to
enable set_log_file as a separate method.
The exploded start() uses twisted.scripts._twistd_unix.UnixAppLogger's
_getLogObserver method as a base.
"""
self.preApplication()
if self.logger._syslog:
self.logger._observer = syslog.SyslogObserver(self._syslogPrefix).emit
elif self.logger._logfilename == '-':
if not self.logger._nodaemon:
sys.exit('Daemons cannot log to stdout, exiting!')
self.logger._observer = log.FileLogObserver(sys.stdout).emit
elif self.logger._nodaemon and not self.logger._logfilename:
self.logger._observer = log.FileLogObserver(sys.stdout).emit
else:
if not self.logger._logfilename:
self.logger._logfilename = 'twistd.log'
self.logger._observer = set_log_file(self.logger._logfilename,
start=False)
log.startLoggingWithObserver(self.logger._observer)
self.logger._initialLog()
self.application = self.createOrGetApplication()
self.postApplication()
self.logger.stop()
def logger():
try:
if 'ANCHORE_LOGFILE' in os.environ:
thefile = os.environ['ANCHORE_LOGFILE']
else:
thefile = "anchore-general.log"
except:
thefile = "anchore-general.log"
f = logfile.LogFile(thefile,
'/var/log/',
rotateLength=10000000,
maxRotatedFiles=10)
log_observer = log.FileLogObserver(f)
return log_observer.emit
def CanvasApp(nginx=False):
application = service.Application("Canvas")
web_server = CanvasTCPServer()
web_server.setServiceParent(application)
log_file = logfile.LogFile(
name="twistd.log",
directory="/var/canvas/website/run",
maxRotatedFiles=100,
)
application.setComponent(log.ILogObserver,
log.FileLogObserver(log_file).emit)
return application
def start(logfile=None, application_name="ooniprobe"):
daily_logfile = None
if not logfile:
logfile = config.basic.logfile
log_folder = os.path.dirname(logfile)
log_filename = os.path.basename(logfile)
daily_logfile = DailyLogFile(log_filename, log_folder)
txlog.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(),
otime.utcPrettyDateNow()))
txlog.startLoggingWithObserver(LogWithNoPrefix(sys.stdout).emit)
txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)
def test_emitNewline(self):
"""
FileLogObserver.emit() will append a newline to its file output.
"""
output = StringIO()
flo = log.FileLogObserver(output)
publisher = log.LogPublisher()
publisher.addObserver(flo.emit)
publisher.msg("Hello!")
result = output.getvalue()
suffix = "Hello!\n"
self.assertTrue(result.endswith(suffix),
"{0!r} does not end with {1!r}".format(result, suffix))
def __init__(self, env, log_file):
self.env = env
if log_file:
log_path = os.path.join(env.getInstancePath(), 'logs')
# clear log files
if env.config.getbool('seishub', 'clear_logs_on_startup'):
log_base = os.path.splitext(log_file)[0]
for file in iglob(os.path.join(log_path, log_base + '.*')):
try:
os.remove(file)
except:
pass
# new log file
daily_log_file = logfile.DailyLogFile(log_file, log_path)
env.app.setComponent(log.ILogObserver,
log.FileLogObserver(daily_log_file).emit)
self.start()
def getLogObserver(verbosity):
baseObserver = log.FileLogObserver(sys.stderr)
if verbosity == 'none':
return lambda ign: None
elif verbosity == 'success':
categories = {'success', 'error'}
elif verbosity == 'error':
categories = {'error'}
else:
raise ValueError('invalid verbosity', verbosity)
def observer(event):
if event['isError'] or event.get('category') in categories:
baseObserver.emit(event)
return observer
def _getLogObserver(logfilename):
"""
Create and return a suitable log observer for the given configuration.
The observer will go to stdout if C{logfilename} is empty or equal to
C{"-"}. Otherwise, it will go to a file with that name.
@type logfilename: C{str}
@param logfilename: The name of the file to which to log, if other than the
default.
@return: An object suitable to be passed to C{log.addObserver}.
"""
if logfilename == '-' or not logfilename:
logFile = sys.stdout
else:
logFile = logfile.LogFile.fromFullPath(logfilename)
return log.FileLogObserver(logFile).emit
def setUp(self):
"""
Add a log observer which records log events in C{self.out}. Also,
make sure the default string encoding is ASCII so that
L{testSingleUnicode} can test the behavior of logging unencodable
unicode messages.
"""
self.out = FakeFile()
self.lp = log.LogPublisher()
self.flo = log.FileLogObserver(self.out)
self.lp.addObserver(self.flo.emit)
try:
str('\N{VULGAR FRACTION ONE HALF}')
except UnicodeEncodeError:
pass
# This is the behavior we want - don't change anything.
self._origEncoding = None
def _getLogObserver(self):
"""
Create and return a suitable log observer for the given configuration.
The observer will go to syslog using the prefix C{_syslogPrefix} if
C{_syslog} is true. Otherwise, it will go to the file named
C{_logfilename} or, if C{_nodaemon} is true and C{_logfilename} is
C{"-"}, to stdout.
@return: An object suitable to be passed to C{log.addObserver}.
"""
if self._syslog:
# FIXME: Requires twisted.python.syslog to be ported to Py3
# https://twistedmatrix.com/trac/ticket/7957
from twisted.python import syslog
return syslog.SyslogObserver(self._syslogPrefix).emit
if self._logfilename == '-':
if not self._nodaemon:
sys.exit('Daemons cannot log to stdout, exiting!')
logFile = sys.stdout
elif self._nodaemon and not self._logfilename:
logFile = sys.stdout
else:
# FIXME: Requires twisted.python.logfile to be ported to Py3
# https://twistedmatrix.com/trac/ticket/6749
from twisted.python import logfile
if not self._logfilename:
self._logfilename = 'twistd.log'
logFile = logfile.LogFile.fromFullPath(self._logfilename)
try:
import signal
except ImportError:
pass
else:
# Override if signal is set to None or SIG_DFL (0)
if not signal.getsignal(signal.SIGUSR1):
def rotateLog(signal, frame):
from twisted.internet import reactor
reactor.callFromThread(logFile.rotate)
signal.signal(signal.SIGUSR1, rotateLog)
return log.FileLogObserver(logFile).emit
def setup(self, daemon, log_syslog, log_file, debug=False):
observer = None
if log_syslog:
observer = syslog.SyslogObserver('siptrackd').emit
elif log_file:
if log_file == '-':
if daemon:
raise SiptrackError('Daemons can\'t log to stdout')
log_fd = sys.stdout
else:
log_fd = logfile.LogFile.fromFullPath(log_file)
observer = log.FileLogObserver(log_fd).emit
else:
raise SiptrackError('No logging method selected')
log.startLoggingWithObserver(observer)
sys.stdout.flush()
self.debug_logging = debug
self.setup_complete = True
return True