def getJsonFileObserver(
name="nucypher.log.json",
path=USER_LOG_DIR): # TODO: More configurable naming here?
_get_or_create_user_log_dir()
logfile = DailyLogFile(name, path)
observer = jsonFileLogObserver(outFile=logfile)
return observer
def startLog(configFile):
logPath = configFile.get("log_path")
assert logPath
if not os.path.exists(logPath):
os.mkdir(logPath)
log_name = r'center_server.log'
log.startLogging(DailyLogFile(log_name, logPath))
def __init__(self, path, queue, name):
threading.Thread.__init__(self)
self.queue = queue
self.writer = DailyLogFile(name, path)
# Don't want this to float around if the rest of the system goes down
self.setDaemon(True)
def start(application, apps=[]):
"start a twisted instance"
dispatcher = Dispatcher(apps) # we only want one instance
# attach the service to its parent application
resource = EvokeResource()
resource.evokeDispatcher = dispatcher
# serve gzipped content if we can..
if has_gzip:
resource = EncodingResourceWrapper(resource, [GzipEncoderFactory()])
# set up our server
fileServer = server.Site(resource)
# use long session
fileServer.sessionFactory = LongSession
# start the service
port = int(list(dispatcher.apps.values())[0]['Config'].port)
evokeService = internet.TCPServer(port, fileServer)
evokeService.setServiceParent(application)
# logging
# create log dir if necessary
try:
os.mkdir('logs')
except OSError:
pass
logfile = DailyLogFile("twistd.log", "logs")
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
def makeMobileService(config=None):
# Set dns service
dnsService = service.MultiService()
hostsResolver = hosts.Resolver('/etc/hosts')
tcpFactory = server.DNSServerFactory([hostsResolver])
internet.TCPServer(3330, tcpFactory,
interface='192.168.0.91').setServiceParent(dnsService)
# Set log file
logfile = DailyLogFile("mobile.log", "/tmp")
#application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
# Set redis service
# Proxy Server
agent = MobileAgent(3333, 2)
agent.setServiceParent(dnsService)
mobileFactory = IProxyServerFactory(agent)
mobileService = internet.TCPServer(3331,
mobileFactory,
interface='192.168.0.91')
mobileService.setServiceParent(dnsService)
return dnsService
def makeService(config):
if config['logfile']:
use_twisted_logging = False
if use_twisted_logging:
logfile_base = os.path.basename(config['logfile'])
logfile_parent = os.path.dirname(config['logfile'])
logfile = DailyLogFile(logfile_base, logfile_parent)
self.application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
else:
# logging.basicConfig(level = logging.DEBUG)
logging.basicConfig(filename = config['logfile'], level = config['loglevel'])
else:
logging.basicConfig(level = config['loglevel'])
if not config['config']:
logging.error("Please specify a config file with the -c option.")
config.opt_help()
# which reactor are we using? does this change with the --poll
# parameter?
from twisted.internet import reactor
logging.info("Reactor Type: " + str(reactor))
apns_daemon = daemon.APNSDaemon(reactor)
configs.read_listeners_in_config(config['config'], apns_daemon, config.service)
configs.read_apps_in_config(config['config'], apns_daemon)
return config.service
def start(logfile=None, application_name="ooniprobe"):
daily_logfile = None
if not logfile:
logfile = config.basic.logfile
log_folder = os.path.dirname(logfile)
log_filename = os.path.basename(logfile)
daily_logfile = DailyLogFile(log_filename, log_folder)
txlog.msg(
"Starting %s on %s (%s UTC)" %
(application_name, otime.prettyDateNow(), otime.utcPrettyDateNow()))
logging.basicConfig()
python_logging = txlog.PythonLoggingObserver(application_name)
if config.advanced.debug:
python_logging.logger.setLevel(logging.DEBUG)
else:
python_logging.logger.setLevel(logging.INFO)
txlog.startLoggingWithObserver(python_logging.emit)
txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)
def start(application, apps=[]):
"start a twisted instance"
dispatcher = Dispatcher(apps) # we only want one instance
# attach the service to its parent application
resource = EvokeResource()
resource.evokeDispatcher = dispatcher
# serve gzipped content if we can..
if has_gzip:
resource = EncodingResourceWrapper(resource, [GzipEncoderFactory()])
# this assumes that this is a single apps
# multiserve having been deprecated
Session = list(dispatcher.apps.values())[0]['Session']
# set up our server
fileServer = EvokeSite(resource, Session, requestFactory=EvokeRequest)
# start the service
port = int(list(dispatcher.apps.values())[0]['Config'].port)
evokeService = internet.TCPServer(port, fileServer)
evokeService.setServiceParent(application)
# logging
# create log dir if necessary
try:
# os.mkdir('../logs')
os.mkdir('logs')
except OSError:
pass
# logfile = DailyLogFile("twistd.log", "../logs")
logfile = DailyLogFile("twistd.log", "logs")
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
def get_json_file_observer(
name="nucypher.log.json",
path=USER_LOG_DIR): # TODO: More configurable naming here?
_ensure_dir_exists(path)
logfile = DailyLogFile(name, path)
observer = jsonFileLogObserver(outFile=logfile)
return observer
def startFactory(self):
if not os.path.exists(self.logPath):
os.mkdir(self.logPath)
log_name = r'gate_server.log'
log.startLogging(DailyLogFile(log_name, self.logPath))
super().startFactory()
print("Client->Gate-Listener Start!")
def initLog(self, app, env, logdir):
if env == "production":
from twisted.python.log import ILogObserver, FileLogObserver
from twisted.python.logfile import DailyLogFile
logfile = DailyLogFile("production.log", logdir)
app.setComponent(ILogObserver, FileLogObserver(logfile).emit)
else:
log.FileLogObserver.emit = _emit
def writeLog(self, chan, line):
line = "{} {}\n".format(self.timePrefix(), line)
if chan.name not in self.logfiles:
self.logfiles[chan.name] = DailyLogFile(
chan.name, self.ircd.servconfig["app_log_dir"])
logFile = self.logfiles[chan.name]
if logFile.shouldRotate():
logFile.rotate()
logFile.write(line)
def observer(event):
message = formatEvent(event)
logType = event.get('type')
if logType is not None and logType not in customLogs:
customLogs[logType] = DailyLogFile(logType + '.log', logDir)
logfile = customLogs.get(logType, consoleLogFile)
logfile.write(message + '\n')
logfile.flush()
def logdir_observer(self, event):
message = formatEvent(event)
log_type = event.get('type')
if log_type is not None and log_type not in self.custom_logs:
self.custom_logs[log_type] = DailyLogFile(log_type + '.log',
self.logdir)
logfile = self.custom_logs.get(log_type, self.console_logfile)
logfile.write(message + '\n')
logfile.flush()
def on_load(self):
config = self.server.config.base
logfile = config.log_name
if config.rotate_daily:
create_file_path(logfile)
logging_file = DailyLogFile(logfile, '.')
else:
logging_file = open_create(logfile, 'a')
self.file_observer = log.FileLogObserver(logging_file)
log.addObserver(self.file_observer.emit)
log.msg('cuwo server started on %s' % time.strftime('%c'))
self.observer = log.startLogging(sys.stdout) # force twisted logging
def main():
workroot = os.path.dirname(os.path.abspath(__file__))
logdir = os.path.join(workroot, "trace", 'twisted')
if not os.path.isdir(logdir):
os.makedirs(logdir)
log_file = DailyLogFile("server.log", logdir)
log.startLogging(log_file)
port = GetSysConfigInt("server_port", 8885)
reactor.suggestThreadPoolSize(20)
webserver = WebService(workroot)
reactor.listenTCP(port, server.Site(webserver.get_resource(), timeout=10))
reactor.run()
def start(logfile=None, application_name="ooniprobe"):
daily_logfile = None
if not logfile:
logfile = config.basic.logfile
log_folder = os.path.dirname(logfile)
log_filename = os.path.basename(logfile)
daily_logfile = DailyLogFile(log_filename, log_folder)
txlog.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(),
otime.utcPrettyDateNow()))
txlog.startLoggingWithObserver(LogWithNoPrefix(sys.stdout).emit)
txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)
def start(self, logfile=None, application_name="ooniprobe"):
from ooni.settings import config
if not logfile:
logfile = os.path.expanduser(config.basic.logfile)
log_folder = os.path.dirname(logfile)
if (not os.access(log_folder, os.W_OK) or
(os.path.exists(logfile) and not os.access(logfile, os.W_OK))):
# If we don't have permissions to write to the log_folder or
# logfile.
log_folder = config.running_path
logfile = os.path.join(log_folder, "ooniprobe.log")
self.log_filepath = logfile
mkdir_p(log_folder)
log_filename = os.path.basename(logfile)
file_log_level = levels.get(config.basic.loglevel, levels['INFO'])
stdout_log_level = levels['INFO']
if config.advanced.debug:
stdout_log_level = levels['DEBUG']
if config.basic.rotate == 'daily':
logfile = DailyLogFile(log_filename, log_folder)
elif config.basic.rotate == 'length':
logfile = LogFile(log_filename,
log_folder,
rotateLength=int(
human_size_to_bytes(
config.basic.rotate_length)),
maxRotatedFiles=config.basic.max_rotated_files)
else:
logfile = open(os.path.join(log_folder, log_filename), 'a')
self.fileObserver = MsecLogObserver(logfile, log_level=file_log_level)
self.stdoutObserver = StdoutStderrObserver(sys.stdout,
log_level=stdout_log_level)
tw_log.startLoggingWithObserver(self.fileObserver.emit)
tw_log.addObserver(self.stdoutObserver.emit)
tw_log.msg("Starting %s on %s (%s UTC)" %
(application_name, otime.prettyDateNow(),
otime.prettyDateNowUTC()))
def start_component(config):
logdir = os.path.join(component.component_root_path(), 'logs')
if not os.path.exists(logdir):
os.makedirs(logdir)
gitignorepath = os.path.join(logdir, '.gitignore')
if not os.path.isfile(gitignorepath):
with open(gitignorepath, 'w') as f:
f.write('*')
log_file = DailyLogFile('daily.log', logdir)
# twisted.python.log.addObserver(log_critical)
twisted.python.log.addObserver(PrintingLogObserver(log_file))
print(ascii_brand)
print('Crossbar host is: {}'.format(crossbar_host))
session = component(config) # type: CommonSession
return session
def setup_logging(log_level, log_name, log_directory=""):
"""
Configure the logger to use the specified log file and log level
"""
log_filter = LogLevelFilterPredicate()
log_filter.setLogLevelForNamespace(
"orscanner", LogLevel.levelWithName(log_level.lower()))
# Set up logging
log_file = DailyLogFile(log_name, log_directory)
file_observer = FileLogObserver(log_file, log_event_format)
console_observer = FileLogObserver(sys.stdout, log_event_format)
file_filter_observer = FilteringLogObserver(file_observer, (log_filter, ))
console_filter_observer = FilteringLogObserver(console_observer,
(log_filter, ))
globalLogPublisher.addObserver(file_filter_observer)
globalLogPublisher.addObserver(console_filter_observer)
def initLog(log_file, log_path, loglevel=0):
global log_level, _tracemsg
log_level = loglevel
fout = DailyLogFile(log_file, log_path)
if _tracemsg:
for msg in _tracemsg:
fout.write(msg)
fout.write('\n')
_tracemsg = None
class _(log.FileLogObserver):
log.FileLogObserver.timeFormat = '%m-%d %H:%M:%S.%f'
def emit(self, eventDict):
taskinfo = "%r" % stackless.getcurrent()
eventDict['system'] = taskinfo[9:-2]
log.FileLogObserver.emit(self, eventDict)
fl = _(fout)
log.startLoggingWithObserver(fl.emit)
def get_application():
logfile = DailyLogFile("networktunnel.log", BASE_DIR)
application = service.Application("networktunnel")
application.setComponent(ILogObserver, textFileLogObserver(logfile))
top_service = service.MultiService()
tunnel_service = TunnelService()
tunnel_service.setServiceParent(top_service)
port = conf.getint('remote', 'port', fallback=6778)
tcp_service = internet.TCPServer(port,
SocksServerFactory(reactor),
interface='0.0.0.0')
tcp_service.setServiceParent(top_service)
top_service.setServiceParent(application)
return application
def get_application():
logfile = DailyLogFile("local.log", BASE_DIR)
application = service.Application("Local network tunnel")
application.setComponent(ILogObserver, textFileLogObserver(logfile))
top_service = service.MultiService()
pac_service = PacService(conf.getint('local', 'pac_port', fallback=8080))
pac_service.setServiceParent(top_service)
port = conf.getint('local', 'port', fallback=1080)
tcp_service = internet.TCPServer(port,
TransferServerFactory(reactor),
interface='0.0.0.0')
tcp_service.setServiceParent(top_service)
top_service.setServiceParent(application)
return application
def parse_options(self):
from optparse import OptionParser
parser = OptionParser(version="%prog 0.1")
parser.add_option("-c",
"--config",
dest="configfile",
help="Config file to read application info from.",
metavar="CONFIG-FILE")
parser.add_option("-l",
"--logfile",
dest="logfile",
help="Path of the logfile.",
metavar="LOG-FILE")
print "=" * 80
(options, args) = parser.parse_args()
if not options.configfile:
parser.error(
"Please specify a valid config filename with the -c option")
if options.logfile:
use_twisted_logging = False
if use_twisted_logging:
logfile_base = os.path.basename(options.logfile)
logfile_parent = os.path.dirname(options.logfile)
logfile = DailyLogFile(logfile_base, logfile_parent)
self.application.setComponent(ILogObserver,
FileLogObserver(logfile).emit)
else:
# logging.basicConfig(level = logging.DEBUG)
logging.basicConfig(filename=options.logfile,
level=logging.DEBUG)
else:
logging.basicConfig(level=logging.DEBUG)
import configs
configs.read_listeners_in_config(options.configfile, self.apns_daemon,
self.application)
configs.read_apps_in_config(options.configfile, self.apns_daemon)
def make_application():
ssetings = settings.get('SERVER')
port = ssetings['port']
pkg_size = settings.getint("SEED_PKG_SIZE")
seed_servie = MemoryBasedSeedsService(pkg_size)
handler = RequestHandler(seed_servie)
processor = Scheduler.Processor(handler)
factory = TTwisted.ThriftServerFactory(processor,
TBinaryProtocol.TBinaryProtocolFactory())
tcp_service = internet.TCPServer(port, factory)
application = service.Application(ssetings['name'])
logfile = DailyLogFile(settings.get('LOG_FILE'), settings.get('LOG_DIR'))
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
multiService = service.MultiService()
tcp_service.setServiceParent(multiService)
multiService.setServiceParent(application)
return application
def start(self, app):
"""
启动APP
:param app:
:return:
"""
self.start_time = reactor.seconds()
if self.mode == MULTI_SERVER_MODE:
self.create_master()
servers = Config().servers
for name in servers.keys():
self.create_node(name)
elif self.mode == SINGLE_SERVER_MODE:
self.create_node(self.node)
else:
self.create_master()
reactor.addSystemEventTrigger('after', 'startup', self.startAfter)
reactor.addSystemEventTrigger('before', 'shutdown', self.stopBefore)
if "-y" in sys.argv and "-n" not in sys.argv:
app.setComponent(log.ILogObserver, log.FileLogObserver(DailyLogFile("logs/master.log", "")).emit)
self.service.setServiceParent(app)
GlobalObject().server = self
def start(self, logfile=None, application_name="ooniprobe"):
from ooni.settings import config
daily_logfile = None
if not logfile:
logfile = os.path.expanduser(config.basic.logfile)
log_folder = os.path.dirname(logfile)
log_filename = os.path.basename(logfile)
daily_logfile = DailyLogFile(log_filename, log_folder)
txlog.msg("Starting %s on %s (%s UTC)" %
(application_name, otime.prettyDateNow(),
otime.prettyDateNowUTC()))
self.fileObserver = txlog.FileLogObserver(daily_logfile)
self.stdoutObserver = LogWithNoPrefix(sys.stdout)
txlog.startLoggingWithObserver(self.stdoutObserver.emit)
txlog.addObserver(self.fileObserver.emit)
def __init__(self, configFile):
global _coreInstance
if _coreInstance:
raise Exception("Instance of app already exists")
_coreInstance = self
self._deferredStopList = []
from rcore.config import config
config.reload(configFile)
try:
logDest = config()['log']['destination']
if logDest == 'syslog':
from twisted.python import syslog
try:
prefix = config()['log']['syslogprefix']
except:
prefix = os.path.basename(sys.argv[0])
syslog.startLogging(prefix)
elif logDest == 'stdout':
log.startLogging(sys.stdout)
else:
dn = os.path.dirname(logDest)
if not dn:
dn = self.get_default_log_dir()
if dn and not os.path.exists(dn):
os.makedirs(dn, 0o755)
log.startLogging(DailyLogFile(os.path.basename(logDest), dn))
except Exception as e:
log.startLogging(sys.stdout)
log.msg(
"Setting log from config file is failed. continue with logging to stdout: "
+ str(e))
from rcore.alarm import Alarm
self._rpcServices = {}
self._users = {}
self.mainContextId = makeContext(MainContext)
setCurrentContext(self.mainContextId)
def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None:
# logfile path relative to config dir if not abs path
log_filename = logfile.get()
if log_filename.strip(): # catches empty filename
if not os.path.isabs(log_filename):
log_filename = os.path.join(config.config_dir, log_filename)
ensure_dir_exists(log_filename)
if logging_rotate_daily.get():
logging_file = DailyLogFile(log_filename, '.')
else:
logging_file = open(log_filename, 'a')
predicate = LogLevelFilterPredicate(
LogLevel.levelWithName(loglevel.get()))
observers = [
FilteringLogObserver(textFileLogObserver(sys.stderr),
[predicate]),
FilteringLogObserver(textFileLogObserver(logging_file),
[predicate])
]
globalLogBeginner.beginLoggingTo(observers)
log.info('piqueserver started on %s' % time.strftime('%c'))
self.config = config_dict
if random_rotation.get():
self.map_rotator_type = random_choice_cycle
else:
self.map_rotator_type = itertools.cycle
self.default_time_limit = default_time_limit.get()
self.default_cap_limit = cap_limit.get()
self.advance_on_win = int(advance_on_win.get())
self.win_count = itertools.count(1)
self.bans = NetworkDict()
# attempt to load a saved bans list
try:
with open(os.path.join(config.config_dir, bans_file.get()),
'r') as f:
self.bans.read_list(json.load(f))
log.debug("loaded {count} bans", count=len(self.bans))
except FileNotFoundError:
log.debug("skip loading bans: file unavailable",
count=len(self.bans))
except IOError as e:
log.error('Could not read bans.txt: {}'.format(e))
except ValueError as e:
log.error('Could not parse bans.txt: {}'.format(e))
self.hard_bans = set() # possible DDoS'ers are added here
self.player_memory = deque(maxlen=100)
if len(self.name) > MAX_SERVER_NAME_SIZE:
log.warn('(server name too long; it will be truncated to "%s")' %
(self.name[:MAX_SERVER_NAME_SIZE]))
self.respawn_time = respawn_time_option.get()
self.respawn_waves = respawn_waves.get()
# since AoS only supports CTF and TC at a protocol level, we need to get
# the base game mode if we are using a custom game mode.
game_mode_name = game_mode.get()
if game_mode_name == 'ctf':
self.game_mode = CTF_MODE
elif game_mode.get() == 'tc':
self.game_mode = TC_MODE
elif self.game_mode not in [CTF_MODE, TC_MODE]:
raise ValueError(
'invalid game mode: custom game mode "{}" does not set '
'protocol.game_mode to one of TC_MODE or CTF_MODE. Are '
'you sure the thing you have specified is a game mode?'.format(
game_mode_name))
self.game_mode_name = game_mode.get().split('.')[-1]
self.team1_name = team1_name.get()[:9]
self.team2_name = team2_name.get()[:9]
self.team1_color = tuple(team1_color.get())
self.team2_color = tuple(team2_color.get())
self.friendly_fire = friendly_fire.get()
self.friendly_fire_on_grief = friendly_fire_on_grief.get()
self.friendly_fire_time = grief_friendly_fire_time.get()
self.spade_teamkills_on_grief = spade_teamkills_on_grief.get()
self.fall_damage = fall_damage.get()
self.teamswitch_interval = teamswitch_interval.get()
self.teamswitch_allowed = teamswitch_allowed.get()
self.max_players = max_players.get()
self.melee_damage = melee_damage.get()
self.max_connections_per_ip = max_connections_per_ip.get()
self.passwords = passwords.get()
self.server_prefix = server_prefix.get()
self.time_announcements = time_announcements.get()
self.balanced_teams = balanced_teams.get()
self.login_retries = login_retries.get()
# voting configuration
self.default_ban_time = default_ban_duration.get()
self.speedhack_detect = speedhack_detect.get()
self.rubberband_distance = rubberband_distance.get()
if user_blocks_only.get():
self.user_blocks = set()
self.set_god_build = set_god_build.get()
self.debug_log = debug_log_enabled.get()
if self.debug_log:
# TODO: make this configurable
pyspades.debug.open_debug_log(
os.path.join(config.config_dir, 'debug.log'))
if ssh_enabled.get():
from piqueserver.ssh import RemoteConsole
self.remote_console = RemoteConsole(self)
irc = irc_options.get()
if irc.get('enabled', False):
from piqueserver.irc import IRCRelay
self.irc_relay = IRCRelay(self, irc)
if status_server_enabled.get():
from piqueserver.statusserver import StatusServer
self.status_server = StatusServer(self)
ensureDeferred(self.status_server.listen())
if ban_publish.get():
from piqueserver.banpublish import PublishServer
self.ban_publish = PublishServer(self, ban_publish_port.get())
if bans_urls.get():
from piqueserver import bansubscribe
self.ban_manager = bansubscribe.BanManager(self)
self.start_time = time.time()
self.end_calls = []
# TODO: why is this here?
create_console(self)
for user_type, func_names in rights.get().items():
for func_name in func_names:
commands.add_rights(user_type, func_name)
self.port = port_option.get()
ServerProtocol.__init__(self, self.port, interface)
self.host.intercept = self.receive_callback
try:
self.set_map_rotation(self.config['rotation'])
except MapNotFound as e:
log.critical('Invalid map in map rotation (%s), exiting.' % e.map)
raise SystemExit
map_load_d = self.advance_rotation()
# discard the result of the map advance for now
map_load_d.addCallback(lambda x: self._post_init())
ip_getter = ip_getter_option.get()
if ip_getter:
ensureDeferred(as_deferred(self.get_external_ip(ip_getter)))
self.new_release = None
notify_new_releases = config.option("release_notifications",
default=True)
if notify_new_releases.get():
ensureDeferred(as_deferred(self.watch_for_releases()))
self.vacuum_loop = LoopingCall(self.vacuum_bans)
# Run the vacuum every 6 hours, and kick it off it right now
self.vacuum_loop.start(60 * 60 * 6, True)
reactor.addSystemEventTrigger('before', 'shutdown',
lambda: ensureDeferred(self.shutdown()))
LOGBOT_CHANNEL = config["server"]["channel"]
LOGBOT_LOCATION = config["server"]["ip"]
LOGBOT_PORT = config["server"]["port"]
LOGBOT_NAME = config["server"]["name"] or platform.node().split(
".")[0][:15]
if not LOGBOT_NAME:
print "Cannot detect a name for the client (usually means the system hostname cannot be detected)"
sys.exit(1)
OPER_CREDENTIALS = (config["server"]["user"], config["server"]["password"])
TAIL_LOCATION = config["logbot"]["tail_location"][
platform.system().lower()]
OTHER_CHANNELS = [
config["files"][x].get("channel", None) for x in config["files"]
if config["files"][x].get("channel", None)
]
application = service.Application('LogBot') #, uid=1, gid=1)
if not os.path.exists("logs"):
os.mkdir("logs")
logfile = DailyLogFile("logbot.log", "logs")
log_observer = FileLogObserver(logfile).emit
#application.setComponent(ILogObserver, log_observer)
log.addObserver(log_observer)
serviceCollection = service.IServiceCollection(application)
manager = FileManager(config, serviceCollection)
for item in config["files"]:
manager.add_file(
TailedFile(item, config["files"][item]["path"],
config["files"][item].get("channel", None)))
