def emit(self,eventDict):
self.write("\033[0m")
if eventDict["isError"]
self.write("\033[91m")
FileLogObserver.emit(self,eventDict)
def emit(self, eventDict):
# Reset text color.
self.write("\033[0m")
if eventDict["isError"]:
# ANSI escape sequence to color text red.
self.write("\033[91m")
FileLogObserver.emit(self, eventDict)
def __init__(self, f=None):
if f is None:
FileLogObserver.__init__(self, sys.stdout)
else:
FileLogObserver.__init__(self, f)
self.timeFormat = "%H:%M:%S"
def __init__(self, f, system=None, override=True):
FileLogObserver.__init__(self, f)
if system:
self._system = system
else:
self._system = "Process {}".format(os.getpid())
self._override = override
def emit(self, eventDict):
# Reset text color.
self.write("\033[0m")
if eventDict["isError"]:
# ANSI escape sequence to color text red.
self.write("\033[91m")
FileLogObserver.emit(self, eventDict)
def __init__(self, f, system = None, override = True):
FileLogObserver.__init__(self, f)
if system:
self._system = system
else:
self._system = "Process {}".format(os.getpid())
self._override = override
def emit(self, eventDict):
if 'system' in eventDict and 'override_system' in eventDict and eventDict['override_system']:
pass
else:
if self._override or (not 'system' in eventDict) or eventDict['system'] == "-":
eventDict['system'] = self._system
FileLogObserver.emit(self, eventDict)
def setup_logfile(self, spider):
logfile = '/Users/batulu/PycharmProjects/spider/douban/log/production_%s.log' % spider.name
fl = FileLogObserver(open(logfile, 'w+'))
fl.start()
# logging.basicConfig(level=logging.INFO, filemode='w', filename='log.txt')
# observer = log.PythonLoggingObserver()
# observer.start()
def emit(self, eventDict):
# Set text color
self.write("\033[0m")
if eventDict["isError"]:
# ANSI escape sequence : red text color
self.write("\033[91m")
FileLogObserver.emit(self, eventDict)
def emit(self, eventDict):
if 'system' in eventDict and 'override_system' in eventDict and eventDict[
'override_system']:
pass
else:
if self._override or (
'system' not in eventDict) or eventDict['system'] == "-":
eventDict['system'] = self._system
FileLogObserver.emit(self, eventDict)
def emit(self, eventDict):
if eventDict['isError']:
# this is executed only if the event is an error
msg = compose_msg(eventDict)
if msg:
# msg is not empty only if the exception is
# from the list of exceptions we are concerned about
msg = bytes(json.dumps(msg), 'utf-8')
sendMessage(msg)
FileLogObserver.emit(self, eventDict)
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(self.filename,
rotateLength=None,
defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def __init__(self, resource):
class FooIO(object):
def write(_, s):
self._lastMessage = s
def flush(_):
pass
FileLogObserver.__init__(self, FooIO())
self.lastMessage = None
self._resource = resource
def __init__(self, resource):
class FooIO(object):
def write(_, s):
self._lastMessage = s
def flush(_):
pass
FileLogObserver.__init__(self, FooIO())
self.lastMessage = None
self._resource = resource
def main(self):
"""Parse arguments and run the script's main function via ``react``."""
observer = None
try:
if not self.log_directory.exists():
self.log_directory.makedirs()
log_path = self.log_directory.child(
b"%s-%d.log" %
(os.path.basename(self.sys_module.argv[0]), os.getpid()))
log_file = log_path.open("a")
observer = FileLogObserver(log_file).emit
addObserver(observer)
msg("Arguments: %s" % (self.sys_module.argv, ))
except (OSError, IOError):
pass
options = self._parse_options(self.sys_module.argv[1:])
# XXX: We shouldn't be using this private _reactor API. See
# https://twistedmatrix.com/trac/ticket/6200 and
# https://twistedmatrix.com/trac/ticket/7527
self._react(self.script.main, (options, ), _reactor=self._reactor)
# Not strictly necessary, but nice cleanup for tests:
if observer is not None:
removeObserver(observer)
log_file.close()
def makeService(config):
if config['logfile']:
use_twisted_logging = False
if use_twisted_logging:
logfile_base = os.path.basename(config['logfile'])
logfile_parent = os.path.dirname(config['logfile'])
logfile = DailyLogFile(logfile_base, logfile_parent)
self.application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
else:
# logging.basicConfig(level = logging.DEBUG)
logging.basicConfig(filename = config['logfile'], level = config['loglevel'])
else:
logging.basicConfig(level = config['loglevel'])
if not config['config']:
logging.error("Please specify a config file with the -c option.")
config.opt_help()
# which reactor are we using? does this change with the --poll
# parameter?
from twisted.internet import reactor
logging.info("Reactor Type: " + str(reactor))
apns_daemon = daemon.APNSDaemon(reactor)
configs.read_listeners_in_config(config['config'], apns_daemon, config.service)
configs.read_apps_in_config(config['config'], apns_daemon)
return config.service
def start(application, apps=[]):
"start a twisted instance"
dispatcher = Dispatcher(apps) # we only want one instance
# attach the service to its parent application
resource = EvokeResource()
resource.evokeDispatcher = dispatcher
# serve gzipped content if we can..
if has_gzip:
resource = EncodingResourceWrapper(resource, [GzipEncoderFactory()])
# set up our server
fileServer = server.Site(resource)
# use long session
fileServer.sessionFactory = LongSession
# start the service
port = int(list(dispatcher.apps.values())[0]['Config'].port)
evokeService = internet.TCPServer(port, fileServer)
evokeService.setServiceParent(application)
# logging
# create log dir if necessary
try:
os.mkdir('logs')
except OSError:
pass
logfile = DailyLogFile("twistd.log", "logs")
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
def get_log_observer():
f = logfile.LogFile("carbon_forwarder.log", log_dir, rotateLength=log_rotate_length, maxRotatedFiles=max_rotated_log_files)
observer = FileLogObserver(f)
filterer = FilteringLogObserver(observer.emit,
[LogLevelFilterPredicate(
LogLevel.levelWithName(log_level))])
return filterer
def start(application, apps=[]):
"start a twisted instance"
dispatcher = Dispatcher(apps) # we only want one instance
# attach the service to its parent application
resource = EvokeResource()
resource.evokeDispatcher = dispatcher
# serve gzipped content if we can..
if has_gzip:
resource = EncodingResourceWrapper(resource, [GzipEncoderFactory()])
# this assumes that this is a single apps
# multiserve having been deprecated
Session = list(dispatcher.apps.values())[0]['Session']
# set up our server
fileServer = EvokeSite(resource, Session, requestFactory=EvokeRequest)
# start the service
port = int(list(dispatcher.apps.values())[0]['Config'].port)
evokeService = internet.TCPServer(port, fileServer)
evokeService.setServiceParent(application)
# logging
# create log dir if necessary
try:
# os.mkdir('../logs')
os.mkdir('logs')
except OSError:
pass
# logfile = DailyLogFile("twistd.log", "../logs")
logfile = DailyLogFile("twistd.log", "logs")
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
def main():
factory = VatsimClientFactory()
log.startLogging(sys.stdout)
#log.addObserver(log.FileLogObserver(open("trace.log",'w')))
addObserver(FileLogObserver(open("trace.log",'w')).emit)
reactor.connectTCP('USA-E.vatsim.net',6809, factory)
reactor.run()
def initLog(self, app, env, logdir):
if env == "production":
from twisted.python.log import ILogObserver, FileLogObserver
from twisted.python.logfile import DailyLogFile
logfile = DailyLogFile("production.log", logdir)
app.setComponent(ILogObserver, FileLogObserver(logfile).emit)
else:
log.FileLogObserver.emit = _emit
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(
self.filename, rotateLength=None, defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def parse_options(self):
from optparse import OptionParser
parser = OptionParser(version="%prog 0.1")
parser.add_option("-c",
"--config",
dest="configfile",
help="Config file to read application info from.",
metavar="CONFIG-FILE")
parser.add_option("-l",
"--logfile",
dest="logfile",
help="Path of the logfile.",
metavar="LOG-FILE")
print "=" * 80
(options, args) = parser.parse_args()
if not options.configfile:
parser.error(
"Please specify a valid config filename with the -c option")
if options.logfile:
use_twisted_logging = False
if use_twisted_logging:
logfile_base = os.path.basename(options.logfile)
logfile_parent = os.path.dirname(options.logfile)
logfile = DailyLogFile(logfile_base, logfile_parent)
self.application.setComponent(ILogObserver,
FileLogObserver(logfile).emit)
else:
# logging.basicConfig(level = logging.DEBUG)
logging.basicConfig(filename=options.logfile,
level=logging.DEBUG)
else:
logging.basicConfig(level=logging.DEBUG)
import configs
configs.read_listeners_in_config(options.configfile, self.apns_daemon,
self.application)
configs.read_apps_in_config(options.configfile, self.apns_daemon)
def make_application():
ssetings = settings.get('SERVER')
port = ssetings['port']
pkg_size = settings.getint("SEED_PKG_SIZE")
seed_servie = MemoryBasedSeedsService(pkg_size)
handler = RequestHandler(seed_servie)
processor = Scheduler.Processor(handler)
factory = TTwisted.ThriftServerFactory(processor,
TBinaryProtocol.TBinaryProtocolFactory())
tcp_service = internet.TCPServer(port, factory)
application = service.Application(ssetings['name'])
logfile = DailyLogFile(settings.get('LOG_FILE'), settings.get('LOG_DIR'))
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
multiService = service.MultiService()
tcp_service.setServiceParent(multiService)
multiService.setServiceParent(application)
return application
class LogService(Service):
name = "log"
def __init__(self, filename):
self.filename = filename
self.logfile = None
self.observer = None
def _signal_handler(self, sig, frame):
reactor.callFromThread(self.logfile.reopen)
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(self.filename,
rotateLength=None,
defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def stopService(self):
Service.stopService(self)
if self.filename != '-':
signal.signal(signal.SIGUSR1, self.__previous_signal_handler)
del self.__previous_signal_handler
self.observer.stop()
self.observer = None
self.logfile.close()
self.logfile = None
else:
self.observer.stop()
self.observer = None
# Don't close stdout.
self.logfile = None
class LogService(Service):
name = "log"
def __init__(self, filename):
self.filename = filename
self.logfile = None
self.observer = None
def _signal_handler(self, sig, frame):
reactor.callFromThread(self.logfile.reopen)
def startService(self):
Service.startService(self)
if self.filename != '-':
self.logfile = LogFile.fromFullPath(
self.filename, rotateLength=None, defaultMode=0o644)
self.__previous_signal_handler = signal.signal(
signal.SIGUSR1, self._signal_handler)
else:
self.logfile = sys.stdout
self.observer = FileLogObserver(self.logfile)
self.observer.start()
def stopService(self):
Service.stopService(self)
if self.filename != '-':
signal.signal(signal.SIGUSR1, self.__previous_signal_handler)
del self.__previous_signal_handler
self.observer.stop()
self.observer = None
self.logfile.close()
self.logfile = None
else:
self.observer.stop()
self.observer = None
# Don't close stdout.
self.logfile = None
def __init__(self, f=None):
if f is None:
FileLogObserver.__init__(self, sys.stdout)
else:
FileLogObserver.__init__(self, f)
def formatEvent(self, event):
self._lastMessage = None
FileLogObserver.emit(self, event)
return self._lastMessage
def checker_master():
if config.LOG_DIRECTORY == "stdout":
log.startLogging(sys.stdout)
else:
log.startLogging(FileLogObserver(daily("checker.log")))
def checker_worker():
if config.LOG_DIRECTORY == "stdout":
log.startLogging(sys.stdout)
else:
log.startLogging(
FileLogObserver(daily("checker-{0}.log".format(config.ARGS.n))))
def make_file_observer(path, log_level_name):
folder, fname = os.path.split(path)
logfile = DailyLogFile(fname, folder)
observer = FileLogObserver(logfile)
return make_wrapped_observer(observer, log_level_name)
SERVER_NAME = 'CHARACTERSERVER'
config.init(SERVER_NAME)
import log
log.init(config.log_threshold)
'''
from syslogger import init_logger
syslog_conf = config.syslog_conf.split(',')
if len(syslog_conf) > 1:
syslog_conf = (syslog_conf[0], int(syslog_conf[1]))
else:
syslog_conf = syslog_conf[0]
init_logger( syslog_conf )
'''
from server import server
from datetime import datetime
reactor.addSystemEventTrigger('before', 'shutdown', server.cleanup)
application = service.Application(SERVER_NAME)
log_path = config.log_path
log_rotate = int(config.log_rotate_interval)
logfile = LogFile('character.log', log_path, rotateLength=log_rotate)
logOb = FileLogObserver(logfile)
logOb.formatTime = lambda when : datetime.fromtimestamp(when).strftime('%m/%d %T.%f')
application.setComponent( ILogObserver, logOb.emit )
internet.TCPServer(config.port, server, interface=config.interface).setServiceParent(service.IServiceCollection(application))
internet.TCPServer(config.adminport, ShellFactory(), interface=config.interface).setServiceParent(service.IServiceCollection(application))
def emit(self, eventDict):
if not "system" in eventDict or eventDict["system"] == "-":
eventDict["system"] = self._system
FileLogObserver.emit(self, eventDict)
def formatEvent(self, event):
self._lastMessage = None
FileLogObserver.emit(self, event)
return self._lastMessage
syslog_conf = (syslog_conf[0], int(syslog_conf[1]))
else:
syslog_conf = syslog_conf[0]
init_logger( syslog_conf )
'''
from server import Server
from datetime import datetime
server = Server()
reactor.addSystemEventTrigger('before', 'shutdown', server.cleanup)
application = service.Application(SERVER_NAME)
log_path = config.log_path
log_rotate = int(config.log_rotate_interval)
logfile = LogFile('message.log', log_path, rotateLength=log_rotate)
logOb = FileLogObserver(logfile)
logOb.formatTime = lambda when: datetime.fromtimestamp(when).strftime(
'%m/%d %T.%f')
application.setComponent(ILogObserver, logOb.emit)
internet.TCPServer(config.port, server,
interface=config.interface).setServiceParent(
service.IServiceCollection(application))
internet.TCPServer(config.adminport,
ShellFactory(),
interface=config.interface).setServiceParent(
service.IServiceCollection(application))
# setup master
basedir = os.path.abspath(os.path.dirname(__file__))
configfile = 'master.cfg'
# Default umask for server
umask = None
# note: this line is matched against to check that this is a buildmaster
# directory; do not edit it.
application = service.Application('buildmaster')
import sys
from twisted.python.log import ILogObserver, FileLogObserver
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
m = BuildMaster(basedir, configfile, umask)
m.setServiceParent(application)
# and slave on the same process!
buildmaster_host = 'localhost'
port = 19989
slavename = 'example-slave'
passwd = 'pass'
keepalive = 600
usepty = 0
umask = None
maxdelay = 300
allow_shutdown = None
def start(self):
FileLogObserver.start(self)
self.client.receive_startup_log(self.module_address, "AMQP log streaming started")
if 'Yes' == service_config.get(service, 'enabled'):
[low_protocol,
low_port] = service_config.get(service, 'low_port').split(':')
[protocol, port] = service_config.get(service, 'port').split(':')
if int(low_port) < 1024:
ipt_file.write('./ipt_set_' + low_protocol + ' ' + low_port +
' ' + port + '\n')
# set file permissin, close, and quit
os.chmod(ipt_file_name, 0744)
ipt_file.close()
quit()
log_file = DailyLogFile(log_file_name, log_path)
file_log_observer = FileLogObserver(log_file)
time_zone = subprocess.check_output(['date', '+%z'])
file_log_observer.timeFormat = "%Y-%m-%d %H:%M:%S,%f," + time_zone.rstrip()
# start logging
log.startLoggingWithObserver(file_log_observer.emit, False)
if 'Yes' == honeypy_config.get('twitter', 'enabled') or \
'Yes' == honeypy_config.get('honeydb', 'enabled') or \
'Yes' == honeypy_config.get('slack', 'enabled') or \
'Yes' == honeypy_config.get('logstash', 'enabled') or \
'Yes' == honeypy_config.get('elasticsearch', 'enabled'):
# tail log file when reactor runs
triageConfig(honeypy_config)
tailer = lib.followtail.FollowTail(log_path + log_file_name)
tailer.lineReceived = triage
def start():
global options, old
args = sys.argv[1:]
options = ClientOptions()
try:
options.parseOptions(args)
except usage.UsageError, u:
print 'ERROR: %s' % u
options.opt_help()
sys.exit(1)
log.startLogging(sys.stderr)
if options['logfile']:
from twisted.python.log import FileLogObserver, addObserver
addObserver(FileLogObserver(open(options['logfile'], 'w')).emit)
if options['host'].find('@') > -1:
options['user'], options['host'] = options['host'].split('@', 1)
if not options['user']:
options['user'] = getuser()
if not options['port']:
options['port'] = '22'
connect(options['host'], options['port'], options)
reactor.run()
print 'Connection to %s closed.' % options['host']
sys.exit()
def api():
if config.LOG_DIRECTORY == "stdout":
log.startLogging(sys.stdout)
else:
log.startLogging(FileLogObserver(daily("api.log")))
log.err('process exit, error[%s]' % result.getErrorMessage())
self._work_d = None
def _readstat(self, statfile):
if self._work_d is None:
self._start_work()
if self._query_protocols and os.path.isfile(statfile):
try:
with open(statfile, 'rb') as f:
self._work_stat = f.read()
except Exception as err:
self._work_stat = err.message
log.err(err.message)
out = [
'run times: %d\n\n' % (self._restart_times), self._work_stat
]
out = ''.join(out)
for protocol in self._query_protocols:
protocol.sendLine(out)
# 创建log目录
if not os.path.isdir('./collectord_log'):
os.mkdir('./collectord_log')
application = service.Application('collectord')
logfile = DailyLogFile('collectord.log', './collectord_log')
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
CollectorServices(31000).setServiceParent(application)
def __init__(self, f=None):
if f is None:
FileLogObserver.__init__(self, sys.stdout)
else:
FileLogObserver.__init__(self, f)
LOGBOT_CHANNEL = config["server"]["channel"]
LOGBOT_LOCATION = config["server"]["ip"]
LOGBOT_PORT = config["server"]["port"]
LOGBOT_NAME = config["server"]["name"] or platform.node().split(
".")[0][:15]
if not LOGBOT_NAME:
print "Cannot detect a name for the client (usually means the system hostname cannot be detected)"
sys.exit(1)
OPER_CREDENTIALS = (config["server"]["user"], config["server"]["password"])
TAIL_LOCATION = config["logbot"]["tail_location"][
platform.system().lower()]
OTHER_CHANNELS = [
config["files"][x].get("channel", None) for x in config["files"]
if config["files"][x].get("channel", None)
]
application = service.Application('LogBot') #, uid=1, gid=1)
if not os.path.exists("logs"):
os.mkdir("logs")
logfile = DailyLogFile("logbot.log", "logs")
log_observer = FileLogObserver(logfile).emit
#application.setComponent(ILogObserver, log_observer)
log.addObserver(log_observer)
serviceCollection = service.IServiceCollection(application)
manager = FileManager(config, serviceCollection)
for item in config["files"]:
manager.add_file(
TailedFile(item, config["files"][item]["path"],
config["files"][item].get("channel", None)))
def utilityMain(
configFileName,
serviceClass,
reactor=None,
serviceMaker=None,
patchConfig=None,
onShutdown=None,
verbose=False,
loadTimezones=False,
):
"""
Shared main-point for utilities.
This function will:
- Load the configuration file named by C{configFileName},
- launch a L{CalDAVServiceMaker}'s with the C{ProcessType} of
C{"Utility"}
- run the reactor, with start/stop events hooked up to the service's
C{startService}/C{stopService} methods.
It is C{serviceClass}'s responsibility to stop the reactor when it's
complete.
@param configFileName: the name of the configuration file to load.
@type configuration: C{str}
@param serviceClass: a 1-argument callable which takes an object that
provides L{ICalendarStore} and/or L{IAddressbookStore} and returns an
L{IService}.
@param patchConfig: a 1-argument callable which takes a config object
and makes and changes necessary for the tool.
@param onShutdown: a 0-argument callable which will run on shutdown.
@param reactor: if specified, the L{IReactorTime} / L{IReactorThreads} /
L{IReactorTCP} (etc) provider to use. If C{None}, the default reactor
will be imported and used.
"""
from calendarserver.tap.caldav import CalDAVServiceMaker, CalDAVOptions
if serviceMaker is None:
serviceMaker = CalDAVServiceMaker
# We want to validate that the actual service is always an instance of WorkerService, so wrap the
# service maker callback inside a function that does that check
def _makeValidService(store):
service = serviceClass(store)
assert isinstance(service, WorkerService)
return service
# Install std i/o observer
if verbose:
observer = StandardIOObserver()
observer.start()
if reactor is None:
from twisted.internet import reactor
try:
config = loadConfig(configFileName)
if patchConfig is not None:
patchConfig(config)
checkDirectories(config)
utilityLogFile = LogFile.fromFullPath(
config.UtilityLogFile,
rotateLength=config.ErrorLogRotateMB * 1024 * 1024,
maxRotatedFiles=config.ErrorLogMaxRotatedFiles,
)
utilityLogObserver = FileLogObserver(utilityLogFile)
utilityLogObserver.start()
config.ProcessType = "Utility"
config.UtilityServiceClass = _makeValidService
autoDisableMemcached(config)
maker = serviceMaker()
# Only perform post-import duties if someone has explicitly said to
maker.doPostImport = getattr(maker, "doPostImport", False)
options = CalDAVOptions
service = maker.makeService(options)
reactor.addSystemEventTrigger("during", "startup", service.startService)
reactor.addSystemEventTrigger("before", "shutdown", service.stopService)
if onShutdown is not None:
reactor.addSystemEventTrigger("before", "shutdown", onShutdown)
if loadTimezones:
TimezoneCache.create()
except (ConfigurationError, OSError), e:
sys.stderr.write("Error: %s\n" % (e,))
return
# setup config parsers
honeypy_config = ConfigParser.ConfigParser()
service_config = ConfigParser.ConfigParser()
# read config files
honeypy_config.read(honeypy_config_file)
service_config.read(service_config_file)
# setup log file and formatting
if not os.path.exists(os.path.dirname(log_path)):
# if log directory does not exist, create it.
os.makedirs(os.path.dirname(log_path))
log_file = DailyLogFile(log_file_name, log_path)
file_log_observer = FileLogObserver(log_file)
time_zone = subprocess.check_output(['date', '+%z'])
file_log_observer.timeFormat = "%Y-%m-%d %H:%M:%S,%f," + time_zone.rstrip()
# start logging
log.startLoggingWithObserver(file_log_observer.emit, False)
if 'Yes' == honeypy_config.get('twitter', 'enabled') or 'Yes' == honeypy_config.get('honeydb', 'enabled'):
# tail log file when reactor runs
triageConfig(honeypy_config)
tailer = lib.followtail.FollowTail(log_path + log_file_name)
tailer.lineReceived = triage
tailer.start()
# services object array
services = []