def setup_logfile(self, spider):
logfile = '/Users/batulu/PycharmProjects/spider/douban/log/production_%s.log' % spider.name
fl = FileLogObserver(open(logfile, 'w+'))
fl.start()
# logging.basicConfig(level=logging.INFO, filemode='w', filename='log.txt')
# observer = log.PythonLoggingObserver()
# observer.start()
class LogService(Service):
name = "log"
def __init__(self, filename):
self.filename = filename
self.logfile = None
self.observer = None
def _signal_handler(self, sig, frame):
reactor.callFromThread(self.logfile.reopen)
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(self.filename,
rotateLength=None,
defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def stopService(self):
Service.stopService(self)
if self.filename != '-':
signal.signal(signal.SIGUSR1, self.__previous_signal_handler)
del self.__previous_signal_handler
self.observer.stop()
self.observer = None
self.logfile.close()
self.logfile = None
else:
self.observer.stop()
self.observer = None
# Don't close stdout.
self.logfile = None
class LogService(Service):
name = "log"
def __init__(self, filename):
self.filename = filename
self.logfile = None
self.observer = None
def _signal_handler(self, sig, frame):
reactor.callFromThread(self.logfile.reopen)
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(
self.filename, rotateLength=None, defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def stopService(self):
Service.stopService(self)
if self.filename != '-':
signal.signal(signal.SIGUSR1, self.__previous_signal_handler)
del self.__previous_signal_handler
self.observer.stop()
self.observer = None
self.logfile.close()
self.logfile = None
else:
self.observer.stop()
self.observer = None
# Don't close stdout.
self.logfile = None
def utilityMain(
configFileName,
serviceClass,
reactor=None,
serviceMaker=None,
patchConfig=None,
onShutdown=None,
verbose=False,
loadTimezones=False,
):
"""
Shared main-point for utilities.
This function will:
- Load the configuration file named by C{configFileName},
- launch a L{CalDAVServiceMaker}'s with the C{ProcessType} of
C{"Utility"}
- run the reactor, with start/stop events hooked up to the service's
C{startService}/C{stopService} methods.
It is C{serviceClass}'s responsibility to stop the reactor when it's
complete.
@param configFileName: the name of the configuration file to load.
@type configuration: C{str}
@param serviceClass: a 1-argument callable which takes an object that
provides L{ICalendarStore} and/or L{IAddressbookStore} and returns an
L{IService}.
@param patchConfig: a 1-argument callable which takes a config object
and makes and changes necessary for the tool.
@param onShutdown: a 0-argument callable which will run on shutdown.
@param reactor: if specified, the L{IReactorTime} / L{IReactorThreads} /
L{IReactorTCP} (etc) provider to use. If C{None}, the default reactor
will be imported and used.
"""
from calendarserver.tap.caldav import CalDAVServiceMaker, CalDAVOptions
if serviceMaker is None:
serviceMaker = CalDAVServiceMaker
# We want to validate that the actual service is always an instance of WorkerService, so wrap the
# service maker callback inside a function that does that check
def _makeValidService(store):
service = serviceClass(store)
assert isinstance(service, WorkerService)
return service
# Install std i/o observer
if verbose:
observer = StandardIOObserver()
observer.start()
if reactor is None:
from twisted.internet import reactor
try:
config = loadConfig(configFileName)
if patchConfig is not None:
patchConfig(config)
checkDirectories(config)
utilityLogFile = LogFile.fromFullPath(
config.UtilityLogFile,
rotateLength=config.ErrorLogRotateMB * 1024 * 1024,
maxRotatedFiles=config.ErrorLogMaxRotatedFiles,
)
utilityLogObserver = FileLogObserver(utilityLogFile)
utilityLogObserver.start()
config.ProcessType = "Utility"
config.UtilityServiceClass = _makeValidService
autoDisableMemcached(config)
maker = serviceMaker()
# Only perform post-import duties if someone has explicitly said to
maker.doPostImport = getattr(maker, "doPostImport", False)
options = CalDAVOptions
service = maker.makeService(options)
reactor.addSystemEventTrigger("during", "startup", service.startService)
reactor.addSystemEventTrigger("before", "shutdown", service.stopService)
if onShutdown is not None:
reactor.addSystemEventTrigger("before", "shutdown", onShutdown)
if loadTimezones:
TimezoneCache.create()
except (ConfigurationError, OSError), e:
sys.stderr.write("Error: %s\n" % (e,))
return
def start(self):
FileLogObserver.start(self)
self.client.receive_startup_log(self.module_address, "AMQP log streaming started")
