def __init__(self, nick, server, channel, port, loggingfile=None):
self.server = server
self.port = port
if loggingfile is not None:
log.startLogging(LogFile.fromFullPath(loggingfile))
self.factory = ConversationBotFactory(self, ConversationBotClient,
nick,
channel)
def __init__(
self, worker_id, path, rotateLength, maxRotatedFiles):
self.worker_id = worker_id
self.path = path
self.rotateLength = rotateLength
self.maxRotatedFiles = maxRotatedFiles
self.closed_count = 0
self.logfile = LogFile.fromFullPath(
path, rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles)
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(
self.filename, rotateLength=None, defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def _openLogFile(self, path):
try:
from twisted.python.logfile import LogFile
log.msg("Setting up http.log rotating %s files of %s bytes each" %
(maxRotatedFiles, rotateLength))
if hasattr(LogFile, "fromFullPath"): # not present in Twisted-2.5.0
return LogFile.fromFullPath(path, rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles)
else:
log.msg("WebStatus: rotated http logs are not supported on this version of Twisted")
except ImportError, e:
log.msg("WebStatus: Unable to set up rotating http.log: %s" % e)
def _openLogFile(self, path):
try:
from twisted.python.logfile import LogFile
log.msg("Setting up http.log rotating %s files of %s bytes each" %
(maxRotatedFiles, rotateLength))
if hasattr(LogFile, "fromFullPath"): # not present in Twisted-2.5.0
return LogFile.fromFullPath(path, rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles)
else:
log.msg("WebStatus: rotated http logs are not supported on this version of Twisted")
except ImportError, e:
log.msg("WebStatus: Unable to set up rotating http.log: %s" % e)
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(self.filename,
rotateLength=None,
defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def __init__(self, logfilename):
if logfilename is None:
logFile = sys.stdout
else:
logFile = LogFile.fromFullPath(logfilename, rotateLength=None)
# Override if signal is set to None or SIG_DFL (0)
if not signal.getsignal(signal.SIGUSR1):
def signalHandler(signal, frame):
from twisted.internet import reactor
reactor.callFromThread(logFile.reopen)
signal.signal(signal.SIGUSR1, signalHandler)
self.observer = log.FileLogObserver(logFile)
def setup_logging():
filename = LOG_SETTINGS['file_path']
log_file = LogFile.fromFullPath(filename,
rotateLength=LOG_SETTINGS['max_bytes'],
maxRotatedFiles=LOG_SETTINGS['max_backups']
) if filename is not None else sys.stdout
log_level = getattr(logging, LOG_SETTINGS['level'])
observer = LevelFileLogObserver(log_file, log_level)
observer.timeFormat = LOG_SETTINGS['time_format']
observer.start()
def setup_logging():
filename = LOG_SETTINGS['file_path']
log_file = LogFile.fromFullPath(
filename,
rotateLength=LOG_SETTINGS['max_bytes'],
maxRotatedFiles=LOG_SETTINGS['max_backups']
) if filename is not None else sys.stdout
log_level = getattr(logging, LOG_SETTINGS['level'])
observer = LevelFileLogObserver(log_file, log_level)
observer.timeFormat = LOG_SETTINGS['time_format']
observer.start()
def opt_logfile(self, logfile_path):
"""
Log to a file. Log is written to ``stdout`` by default. The logfile
directory is created if it does not already exist.
"""
logfile = FilePath(logfile_path)
logfile_directory = logfile.parent()
if not logfile_directory.exists():
logfile_directory.makedirs()
self['logfile'] = LogFile.fromFullPath(
logfile.path,
rotateLength=LOGFILE_LENGTH,
maxRotatedFiles=LOGFILE_COUNT,
)
def setServiceParent(self, app):
MultiService.setServiceParent(self, app)
if config.ErrorLogEnabled:
errorLogFile = LogFile.fromFullPath(
config.ErrorLogFile,
rotateLength=config.ErrorLogRotateMB * 1024 * 1024,
maxRotatedFiles=config.ErrorLogMaxRotatedFiles
)
errorLogObserver = FileLogObserver(errorLogFile).emit
# Registering ILogObserver with the Application object
# gets our observer picked up within AppLogger.start( )
app.setComponent(ILogObserver, errorLogObserver)
def opt_logfile(self, logfile_path):
"""
Log to a file. Log is written to ``stdout`` by default. The logfile
directory is created if it does not already exist.
"""
logfile = FilePath(logfile_path)
logfile_directory = logfile.parent()
if not logfile_directory.exists():
logfile_directory.makedirs()
self['logfile'] = LogFile.fromFullPath(
logfile.path,
rotateLength=LOGFILE_LENGTH,
maxRotatedFiles=LOGFILE_COUNT,
)
def makeService(config):
# install simple, blocking DNS resolver.
from twisted.internet import reactor
from twisted.internet.base import BlockingResolver
reactor.installResolver(BlockingResolver())
try:
# Create dbus mainloop
import dbus.mainloop.glib
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
except ImportError:
warnings.warn('Failed to import the dbus module, some functionality might not work.')
# Check if it is the right thing
if not hasattr(config.module, 'Application'):
raise usage.UsageError("Invalid application module: " + config.appName)
# Instantiate the main application
s = config.module.Application(config.appName, config.opts, config.appOpts)
# Set quitflag
s.quitFlag = launcher.QuitFlag(s.path("temp").child("quitflag"))
# Set the name
s.setName(config.appName)
# make sure the relevant paths exist
for kind in ["temp", "db"]:
path = s.path(kind)
if not path.exists():
path.createDirectory()
# Set up logging
logFile = s.path("logfile")
if not logFile.parent().exists():
logFile.parent().createDirectory()
filename = s.path("logfile").path
if config.opts['no-logrotate']:
observer = RotatableFileLogObserver(filename)
else:
logFile = LogFile.fromFullPath(filename, maxRotatedFiles=9)
observer = log.FileLogObserver(logFile).emit
log.addObserver(observer)
return s
def postOptions(self):
if self["journald"]:
destination = JournaldDestination()
else:
if self["logfile"] is None:
logfile = self._sys_module.stdout
else:
logfilepath = FilePath(self["logfile"])
logfilepath_directory = logfilepath.parent()
if not logfilepath_directory.exists():
logfilepath_directory.makedirs()
# A twisted.python.logfile which has write and flush methods
# but which also rotates the log file.
logfile = LogFile.fromFullPath(
logfilepath.path, rotateLength=LOGFILE_LENGTH, maxRotatedFiles=LOGFILE_COUNT
)
destination = FileDestination(file=logfile)
self.eliot_destination = destination
original_postOptions(self)
def postOptions(self):
if self['journald']:
destination = JournaldDestination()
else:
if self['logfile'] is None:
logfile = self._sys_module.stdout
else:
logfilepath = FilePath(self['logfile'])
logfilepath_directory = logfilepath.parent()
if not logfilepath_directory.exists():
logfilepath_directory.makedirs()
# A twisted.python.logfile which has write and flush methods
# but which also rotates the log file.
logfile = LogFile.fromFullPath(
logfilepath.path,
rotateLength=LOGFILE_LENGTH,
maxRotatedFiles=LOGFILE_COUNT,
)
destination = FileDestination(file=logfile)
self.eliot_destination = destination
original_postOptions(self)
def __init__(self, serverPushCb, queue=None, path=None, filter=True,
bufferDelay=1, retryDelay=5, blackList=None, filterFunc=None):
"""
@serverPushCb: callback to be used. It receives 'self' as parameter. It
should call self.queueNextServerPush() when it's done to queue the next
push. It is guaranteed that the queue is not empty when this function is
called.
@queue: a item queue that implements IQueue.
@path: path to save config.
@filter: when True (default), removes all "", None, False, [] or {}
entries.
@bufferDelay: amount of time events are queued before sending, to
reduce the number of push requests rate. This is the delay between the
end of a request to initializing a new one.
@retryDelay: amount of time between retries when no items were pushed on
last serverPushCb call.
@blackList: events that shouldn't be sent.
@filterFunc: optional function applied to items added to packet payload
"""
StatusReceiverMultiService.__init__(self)
# Parameters.
self.queue = queue
if self.queue is None:
self.queue = MemoryQueue()
self.queue = IndexedQueue(self.queue)
self.path = path
self.filter = filter
self.bufferDelay = bufferDelay
self.retryDelay = retryDelay
if not callable(serverPushCb):
raise NotImplementedError('Please pass serverPushCb parameter.')
def hookPushCb():
# Update the index so we know if the next push succeed or not, don't
# update the value when the queue is empty.
if not self.queue.nbItems():
return
self.lastIndex = self.queue.getIndex()
return serverPushCb(self)
self.serverPushCb = hookPushCb
self.blackList = blackList
self.filterFunc = filterFunc
# Other defaults.
# IDelayedCall object that represents the next queued push.
self.task = None
self.stopped = False
self.lastIndex = -1
self.state = {}
self.state['started'] = str(datetime.datetime.utcnow())
self.state['next_id'] = 1
self.state['last_id_pushed'] = 0
# Try to load back the state.
if self.path and os.path.isdir(self.path):
state_path = os.path.join(self.path, 'state')
if os.path.isfile(state_path):
self.state.update(json.load(open(state_path, 'r')))
if self.queue.nbItems():
# Last shutdown was not clean, don't wait to send events.
self.queueNextServerPush()
self.verboseLog = LogFile.fromFullPath(
'status_push.log', rotateLength=10*1024*1024, maxRotatedFiles=14)
wsgi_root = wsgi_resource()
root = twresource.Root(wsgi_root)
# Servce Django media files off of /media:
staticrsrc = static.File(os.path.join(DJANGO_PROJECT_PATH, "Weblvn/media"))
root.putChild("media", staticrsrc)
# The cool part! Add in pure Twisted Web Resouce in the mix
# This 'pure twisted' code could be using twisted's XMPP functionality, etc:
#root.putChild("google", twresource.GoogleResource())
# my add (from buildbot runner.py):
rotateLength = 10000000
maxRotatedFiles = 5
try:
from twisted.python.logfile import LogFile
from twisted.python.log import ILogObserver, FileLogObserver
logfile = LogFile.fromFullPath(TWISTED_LOG, rotateLength=rotateLength,
maxRotatedFiles=maxRotatedFiles)
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
except ImportError:
# probably not yet twisted 8.2.0 and beyond, can't set log yet
pass
# Serve it up:
main_site = server.Site(root)
internet.TCPServer(PORT, main_site).setServiceParent(application)
def create_logfile(self):
'''Creates and returns a temporary LogFile.'''
return LogFile.fromFullPath(self.mktemp())
from django.conf import settings
sys.path.insert(0, settings.APPS_ROOT)
from twisted.application import service
from twisted.python.log import ILogObserver
from twisted.python.logfile import LogFile
from tx_logging.observers import LevelFileLogObserver
from commander.service import RootService
from commander.settings import COMMANDER_LOG as LOG_SETTINGS
# Init logging ----------------------------------------------------------------
filename = LOG_SETTINGS['filename']
log_file = LogFile.fromFullPath(filename,
rotateLength=LOG_SETTINGS['maxBytes'],
maxRotatedFiles=LOG_SETTINGS['backupCount']
) if filename is not None else sys.stdout
log_level = getattr(logging, LOG_SETTINGS['level'])
observer = LevelFileLogObserver(log_file, log_level)
observer.timeFormat = LOG_SETTINGS['timeFormat']
# Init application ------------------------------------------------------------
application = service.Application("IL-2 Events Commander")
application.setComponent(ILogObserver, observer.emit)
# Init commander service ------------------------------------------------------
root_service = RootService()
root_service.setServiceParent(application)
def __init__(self,
serverPushCb,
queue=None,
path=None,
filter=True,
bufferDelay=1,
retryDelay=5,
blackList=None,
filterFunc=None):
"""
@serverPushCb: callback to be used. It receives 'self' as parameter. It
should call self.queueNextServerPush() when it's done to queue the next
push. It is guaranteed that the queue is not empty when this function is
called.
@queue: a item queue that implements IQueue.
@path: path to save config.
@filter: when True (default), removes all "", None, False, [] or {}
entries.
@bufferDelay: amount of time events are queued before sending, to
reduce the number of push requests rate. This is the delay between the
end of a request to initializing a new one.
@retryDelay: amount of time between retries when no items were pushed on
last serverPushCb call.
@blackList: events that shouldn't be sent.
@filterFunc: optional function applied to items added to packet payload
"""
StatusReceiverMultiService.__init__(self)
# Parameters.
self.queue = queue
if self.queue is None:
self.queue = MemoryQueue()
self.queue = IndexedQueue(self.queue)
self.path = path
self.filter = filter
self.bufferDelay = bufferDelay
self.retryDelay = retryDelay
if not callable(serverPushCb):
raise NotImplementedError('Please pass serverPushCb parameter.')
def hookPushCb():
# Update the index so we know if the next push succeed or not, don't
# update the value when the queue is empty.
if not self.queue.nbItems():
return
self.lastIndex = self.queue.getIndex()
return serverPushCb(self)
self.serverPushCb = hookPushCb
self.blackList = blackList
self.filterFunc = filterFunc
# Other defaults.
# IDelayedCall object that represents the next queued push.
self.task = None
self.stopped = False
self.lastIndex = -1
self.state = {}
self.state['started'] = str(datetime.datetime.utcnow())
self.state['next_id'] = 1
self.state['last_id_pushed'] = 0
# Try to load back the state.
if self.path and os.path.isdir(self.path):
state_path = os.path.join(self.path, 'state')
if os.path.isfile(state_path):
self.state.update(json.load(open(state_path, 'r')))
if self.queue.nbItems():
# Last shutdown was not clean, don't wait to send events.
self.queueNextServerPush()
self.verboseLog = LogFile.fromFullPath('status_push.log',
rotateLength=10 * 1024 * 1024,
maxRotatedFiles=14)
from twisted.application import service
from twisted.python.log import ILogObserver
from twisted.python.logfile import LogFile
from tx_logging.observers import LevelFileLogObserver
from commander.service import RootService
from commander.settings import COMMANDER_LOG as LOG_SETTINGS
# Init logging ----------------------------------------------------------------
filename = LOG_SETTINGS['filename']
log_file = LogFile.fromFullPath(
filename,
rotateLength=LOG_SETTINGS['maxBytes'],
maxRotatedFiles=LOG_SETTINGS['backupCount']
) if filename is not None else sys.stdout
log_level = getattr(logging, LOG_SETTINGS['level'])
observer = LevelFileLogObserver(log_file, log_level)
observer.timeFormat = LOG_SETTINGS['timeFormat']
# Init application ------------------------------------------------------------
application = service.Application("IL-2 Events Commander")
application.setComponent(ILogObserver, observer.emit)
# Init commander service ------------------------------------------------------
def _openLogFile(self, path):
self._nativeize = True
return LogFile.fromFullPath(
path, rotateLength=self.rotateLength, maxRotatedFiles=self.maxRotatedFiles)
def initialize_twisted_logging(self):
twenty_megabytes = 20000000
log_file = LogFile.fromFullPath(self.configuration['log_filename'],
maxRotatedFiles=10,
rotateLength=twenty_megabytes)
log.startLogging(log_file)
def initialize_twisted_logging(self):
twenty_megabytes = 20000000
log_file = LogFile.fromFullPath(self.configuration['log_filename'],
maxRotatedFiles=10,
rotateLength=twenty_megabytes)
log.startLogging(log_file)
def utilityMain(
configFileName, serviceClass, reactor=None, serviceMaker=None,
patchConfig=None, onShutdown=None, verbose=False, loadTimezones=False
):
"""
Shared main-point for utilities.
This function will:
- Load the configuration file named by C{configFileName},
- launch a L{CalDAVServiceMaker}'s with the C{ProcessType} of
C{"Utility"}
- run the reactor, with start/stop events hooked up to the service's
C{startService}/C{stopService} methods.
It is C{serviceClass}'s responsibility to stop the reactor when it's
complete.
@param configFileName: the name of the configuration file to load.
@type configuration: C{str}
@param serviceClass: a 1-argument callable which takes an object that
provides L{ICalendarStore} and/or L{IAddressbookStore} and returns an
L{IService}.
@param patchConfig: a 1-argument callable which takes a config object
and makes and changes necessary for the tool.
@param onShutdown: a 0-argument callable which will run on shutdown.
@param reactor: if specified, the L{IReactorTime} / L{IReactorThreads} /
L{IReactorTCP} (etc) provider to use. If C{None}, the default reactor
will be imported and used.
"""
from calendarserver.tap.caldav import CalDAVServiceMaker, CalDAVOptions
if serviceMaker is None:
serviceMaker = CalDAVServiceMaker
# We want to validate that the actual service is always an instance of WorkerService, so wrap the
# service maker callback inside a function that does that check
def _makeValidService(store):
service = serviceClass(store)
assert isinstance(service, WorkerService)
return service
# Install std i/o observer
observers = []
if verbose:
observers.append(FileLogObserver(sys.stdout, lambda event: formatEventAsClassicLogText(event)))
if reactor is None:
from twisted.internet import reactor
try:
config = loadConfig(configFileName)
if patchConfig is not None:
patchConfig(config)
checkDirectories(config)
utilityLogFile = LogFile.fromFullPath(
config.UtilityLogFile,
rotateLength=config.ErrorLogRotateMB * 1024 * 1024,
maxRotatedFiles=config.ErrorLogMaxRotatedFiles
)
utilityLogObserver = FileLogObserver(utilityLogFile, lambda event: formatEventAsClassicLogText(event))
utilityLogObserver._encoding = "utf-8"
observers.append(utilityLogObserver)
Logger.beginLoggingTo(observers, redirectStandardIO=False)
config.ProcessType = "Utility"
config.UtilityServiceClass = _makeValidService
autoDisableMemcached(config)
maker = serviceMaker()
# Only perform post-import duties if someone has explicitly said to
maker.doPostImport = getattr(maker, "doPostImport", False)
options = CalDAVOptions
service = maker.makeService(options)
reactor.addSystemEventTrigger("during", "startup", service.startService)
reactor.addSystemEventTrigger("before", "shutdown", service.stopService)
if onShutdown is not None:
reactor.addSystemEventTrigger("before", "shutdown", onShutdown)
if loadTimezones:
TimezoneCache.create()
except (ConfigurationError, OSError), e:
sys.stderr.write("Error: %s\n" % (e,))
return
def _openLogFile(self, path):
return LogFile.fromFullPath(path,
rotateLength=self.rotateLength,
maxRotatedFiles=self.maxRotatedFiles)
from twisted.python import log
from twisted.web import static, server
from twisted.web.resource import Resource
from twisted.python.logfile import LogFile
try:
from autobahn.wamp import WampServerFactory
except ImportError: # autobahn 0.8.0+
from autobahn.wamp1.protocol import WampServerFactory
sys.path.append("/etc/yadtbroadcast-server")
from broadcastserverconfig import LOG_FILE, CACHE_FILE, WS_PORT, DOCROOT_DIR, HTTP_PORT
log.startLogging(LogFile.fromFullPath(LOG_FILE))
import yadtbroadcastserver
try:
os.makedirs(os.path.dirname(CACHE_FILE))
except exceptions.OSError, e:
if e.errno != 17:
log.err()
try:
os.makedirs(os.path.dirname(LOG_FILE))
except exceptions.OSError, e:
if e.errno != 17:
log.err()
# TODO refactor: use util method in ws lib for url creation
from twisted.protocols.amp import AMP, Command, Integer, String
from twisted.python.usage import Options, UsageError
from twisted.internet import reactor, task, utils
from twisted.internet.protocol import ServerFactory
import sys,re
from twisted.application import service, internet
from simplebb.server import main
rotateLength = 1000000
maxRotatedFiles = 5
from twisted.python.logfile import LogFile
from twisted.python.log import ILogObserver, FileLogObserver
application = service.Application("My app")
logfile = LogFile.fromFullPath("server.log", rotateLength=rotateLength,
maxRotatedFiles=maxRotatedFiles)
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
service.IProcess(application).processName = "simplebbsd"
s = main(use_tac=True)
s.setServiceParent(application)
def utilityMain(
configFileName,
serviceClass,
reactor=None,
serviceMaker=None,
patchConfig=None,
onShutdown=None,
verbose=False,
loadTimezones=False,
):
"""
Shared main-point for utilities.
This function will:
- Load the configuration file named by C{configFileName},
- launch a L{CalDAVServiceMaker}'s with the C{ProcessType} of
C{"Utility"}
- run the reactor, with start/stop events hooked up to the service's
C{startService}/C{stopService} methods.
It is C{serviceClass}'s responsibility to stop the reactor when it's
complete.
@param configFileName: the name of the configuration file to load.
@type configuration: C{str}
@param serviceClass: a 1-argument callable which takes an object that
provides L{ICalendarStore} and/or L{IAddressbookStore} and returns an
L{IService}.
@param patchConfig: a 1-argument callable which takes a config object
and makes and changes necessary for the tool.
@param onShutdown: a 0-argument callable which will run on shutdown.
@param reactor: if specified, the L{IReactorTime} / L{IReactorThreads} /
L{IReactorTCP} (etc) provider to use. If C{None}, the default reactor
will be imported and used.
"""
from calendarserver.tap.caldav import CalDAVServiceMaker, CalDAVOptions
if serviceMaker is None:
serviceMaker = CalDAVServiceMaker
# We want to validate that the actual service is always an instance of WorkerService, so wrap the
# service maker callback inside a function that does that check
def _makeValidService(store):
service = serviceClass(store)
assert isinstance(service, WorkerService)
return service
# Install std i/o observer
if verbose:
observer = StandardIOObserver()
observer.start()
if reactor is None:
from twisted.internet import reactor
try:
config = loadConfig(configFileName)
if patchConfig is not None:
patchConfig(config)
checkDirectories(config)
utilityLogFile = LogFile.fromFullPath(
config.UtilityLogFile,
rotateLength=config.ErrorLogRotateMB * 1024 * 1024,
maxRotatedFiles=config.ErrorLogMaxRotatedFiles,
)
utilityLogObserver = FileLogObserver(utilityLogFile)
utilityLogObserver.start()
config.ProcessType = "Utility"
config.UtilityServiceClass = _makeValidService
autoDisableMemcached(config)
maker = serviceMaker()
# Only perform post-import duties if someone has explicitly said to
maker.doPostImport = getattr(maker, "doPostImport", False)
options = CalDAVOptions
service = maker.makeService(options)
reactor.addSystemEventTrigger("during", "startup", service.startService)
reactor.addSystemEventTrigger("before", "shutdown", service.stopService)
if onShutdown is not None:
reactor.addSystemEventTrigger("before", "shutdown", onShutdown)
if loadTimezones:
TimezoneCache.create()
except (ConfigurationError, OSError), e:
sys.stderr.write("Error: %s\n" % (e,))
return
def _openLogFile(self, path):
self._nativeize = True
return LogFile.fromFullPath(path,
rotateLength=self.rotateLength,
maxRotatedFiles=self.maxRotatedFiles)
def _openLogFile(self, path):
return LogFile.fromFullPath(
path, rotateLength=self.rotateLength, maxRotatedFiles=self.maxRotatedFiles)
