def _daemonize(self):
#脱离父进程
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
Loger().ERROR("Scoutd fork #1 failed:"+str(e.strerror))
sys.exit(1)
os.setsid()
os.chdir(self.chdir)
os.umask(0)
#第二次fork，禁止进程重新打开控制终端
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
Loger().ERROR("Scoutd fork #2 failed:"+str(e.strerror))
sys.exit(1)
sys.stdout.flush()
sys.stderr.flush()
si = file(self.stdin, 'r')
so = file(self.stdout, 'a+')
se = file(self.stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
atexit.register(self.delpid)
pid = str(os.getpid())
file(self.pidfile,'w+').write("%s\n" % pid)
def stop(self):
"""
Stop the daemon
"""
self._stop_first()
try:
pf = file(self.pidfile,'r')
pid = int(pf.read().strip())
pf.close()
except IOError as e:
pid = None
if not pid:
message = "pidfile %s does not exist. Scoutd not running?"
Loger().ERROR(message % self.pidfile)
return
try:
while 1:
os.kill(pid, SIGTERM)
time.sleep(0.1)
except OSError as err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
Loger().WARNING("stop Scoutd Success.")
else:
Loger().ERROR("stop error,"+str(err))
sys.exit(1)
def rule_filter(self, parse={}):
if parse['bolt'] in ["TCP", "UDP"]:
col = self.cache_connect(parse['bolt'])
else:
Loger().ERROR("Bolt value must be 'TCP', 'UDP' !")
raise
# 解析filter，注意配置文件不能缺少关键字
try:
timeDelta = parse['filter']['timeDelta'] #时间区间, Seconds.
trustIps = parse['filter']['trustIps'] #排除src白名单
motrPort = parse['filter']['motrPort'] #过滤端口
motrProto = parse['filter']['motrProto'] #过滤协议
flags = parse['filter']['flags'] #连接状态
noOfConnections = parse['filter']['noOfConnections'] #阀值
noOfCondition = parse['filter']['noOfCondition'] #阀值条件 如$ge\$gt\$gte\$lt\$lte
returnFiled = parse['filter']['returnFiled'] #过滤器返回的字段名, blot表里必须存在
except Exception as e:
Loger().ERROR("filter rule farmat error.")
raise
#构造查询
aggs=[]
lte_time = int(time.time())
gte_time = (lte_time - timeDelta)
if timeDelta: aggs.append({'$match': {'time' : {'$gte' : gte_time, '$lte' : lte_time}}})
if flags: aggs.append({'$match': {'flags': {'$in': flags}}})
if motrPort: aggs.append({'$match': {'dport': {'$in': motrPort}}})
if trustIps: aggs.append({'$match': {'src': {'$nin': trustIps}}})
aggs.append({'$group': {'_id': '$%s' %returnFiled, 'total': {'$sum': 1}}})
aggs.append({'$match': {'total': {noOfCondition: noOfConnections}}})
#Loger().DEBUG(aggs)
return CacheServer().find_aggregate(col, aggs)
def rule_unblock(self, confname, parse={}):
# 解析block，注意配置文件不能缺少关键字
try:
action = parse['block']['action']
expire = parse['block']['expire']
command = parse['block']['ubkcmd']
iptables = parse['block']['iptables']
except Exception as e:
Loger().ERROR("block rule farmat error.")
raise
if action:
#解封过时记录
call_cmd=''
kwargs={'exptime': {'$lt': int(time.time())}, 'confname': confname}
for item in CacheServer().find_conditions(self.Bcol, **kwargs):
if iptables:
call_cmd=("/sbin/iptables -D INPUT -s %s -j DROP" % item['_id'])
else:
if command.find(' %s')>0:
temp={}
temp['_id']=item['_id']
temp['total']=item['total']
temp['unblock']=1
call_cmd=(command % temp)
subprocess.call(call_cmd, shell=True)
Loger().WARNING(Notes['UNLOCK'] % item['_id'])
CacheServer().delete_many(self.Bcol, kwargs)
def _run(self):
if not cacheserver_running_alive():
Loger().ERROR('CacheServer not running... you must be start it first!')
sys.exit(1)
Loger().INFO('Scoutd %s ' % ScoutBase().avr['version'])
Loger().INFO('Copyright (C) 2011-2019, YWJT.org.')
Loger().INFO('Scoutd started with pid %d' % os.getpid())
Loger().INFO('Scoutd started with %s' % datetime.datetime.now().strftime("%m/%d/%Y %H:%M"))
Scout().run()
def sendto(self, subject, msg, receiver):
if self.avr['smtp_ssl']:
try:
self.sslsend(subject, msg, receiver)
Loger().WARNING('[MAIL] Send mail Success.')
except Exception as e:
Loger().ERROR('[MAIL] Send mail failed to: %s' % e)
else:
try:
self.nonsend(subject, msg, receiver)
Loger().WARNING('[MAIL] Send mail Success.')
except Exception as e:
Loger().ERROR('[MAIL] Send mail failed to: %s' % e)
def stop(self):
try:
if cacheserver_running_alive():
os.chdir(PROC_DIR)
status, output = commands.getstatusoutput(
self.__CacheStopCommand)
if status == 0:
Loger().WARNING('CacheServer stop Success. {}\n'.format(
time.ctime()))
time.sleep(2)
else:
Loger().ERROR('CacheServer stop fail! %s' % output)
raise
except Exception as e:
pass
def __init__(self):
ScoutBase.__init__(self)
__dbPath = '%s/.Scoutd/Bolt' % CACHE_DIR
__logPath = '%s/cacheserver.log' % LOGS_DIR
__storagePort = 6666
__storageSize = self.avr['storage_size']
if self.avr['storage_type'] in ['Memory', 'Disk']:
if self.avr['storage_type'] == 'Memory':
self.__CacheRunCommand = 'cacheServer \
--port=%d \
--dbpath=%s \
--storageEngine=inMemory \
--inMemorySizeGB=%d \
--logpath=%s \
--logappend \
--fork \
--quiet' % (__storagePort, __dbPath, __storageSize, __logPath)
else:
self.__CacheRunCommand = 'cacheServer \
--port=%d \
--dbpath=%s \
--storageEngine=wiredTiger \
--wiredTigerCacheSizeGB=%d \
--logpath=%s \
--logappend \
--fork \
--quiet' % (__storagePort, __dbPath, __storageSize, __logPath)
self.__CacheStopCommand = 'cacheServer --dbpath=%s --shutdown' % __dbPath
else:
Loger().CRITICAL(
"'storage_type' value not match! options: 'Memory' or 'Disk'")
raise
def start(self):
try:
if not cacheserver_running_alive():
os.chdir(PROC_DIR)
status, output = commands.getstatusoutput(
self.__CacheRunCommand)
if status == 0:
Loger().INFO('CacheServer started with pid {}\n'.format(
os.getpid()))
time.sleep(2)
else:
Loger().ERROR('CacheServer started failed.')
else:
Loger().INFO('CacheServer Daemon Alive!')
except Exception as e:
sys.stdout.write(str(e) + '\n')
pass
def saveCache(self, bolt, stdout):
try:
obj = getattr(self, str(bolt))
if type(stdout) == list:
CacheServer().insert_many(obj, stdout)
else:
CacheServer().insert_one(obj, stdout)
except Exception as e:
Loger().ERROR("no collection name is %s, Error: %s" %
(str(stdout["proto"]), str(e)))
pass
def rule_notice(self, confname, parse={}, data={}):
try:
send = parse['notice']['send']
email = parse['notice']['email']
except Exception as e:
Loger().ERROR("block rule farmat error.")
raise
subject = "Scout email server"
if send:
for receiver in email:
PyEmail().sendto(subject, Notes['MAIL'] %(data['_id'], confname, data['total']), receiver)
def rule_block(self, confname, parse={}, data={}):
# 输出符合过滤条件的信息
Loger().WARNING("[%s] %s" % (confname, data))
# 解析block，注意配置文件不能缺少关键字
try:
action = parse['block']['action']
expire = parse['block']['expire']
command = parse['block']['blkcmd']
iptables = parse['block']['iptables']
except Exception as e:
Loger().ERROR("block rule farmat error.")
raise
block_data={}
block_data['_id'] = data['_id']
block_data['total'] = data['total']
block_data["time"]=int(time.time())
block_data["exptime"]=int(time.time()) + expire
block_data["confname"]=str(confname)
block_data["command"]=''
if action:
if iptables:
block_data["command"] = ("/sbin/iptables -I INPUT -s %s -j DROP" % data['_id'])
else:
if command.find(' %s')>0:
data['block']=1
block_data["command"] = (command % data)
state=self.load_cache(self.Bcol, {'_id': data['_id']}, block_data)
if state:
subprocess.call(block_data["command"], shell=True)
Loger().WARNING(Notes['LOCK'] % (data['_id'], data['total']))
#发出邮件通知
self.rule_notice(confname, parse, data)
else:
Loger().WARNING(Notes['RECORD'] % (data['_id'], data['total']))
def start(self): """ Start the daemon """ try: pf = file(self.pidfile,'r') pid = int(pf.read().strip()) pf.close() except IOError as e: pid = None if pid: message = "Start error,pidfile %s already exist. Scoutd already running?" Loger().ERROR(message % self.pidfile) sys.exit(1) self._daemonize() self._run()
def LOOP(self, keeprunning=True, timeout=1):
while keeprunning:
"""如果缓存里有key==>value
则取缓存数据,否则重新加载配置文件
"""
if self.S:
for k in self.S.keys():
#执行解锁
self.rule_unblock(k, self.S[k])
#执行封锁
for res in self.rule_filter(self.S[k]):
self.rule_block(k, self.S[k], res)
Loger().WARNING("[%s.%s] %s" % (k, self.filetype, res))
else:
ptn = re.compile('.*\.%s' % self.filetype)
for f in os.listdir(self.filepath):
ff = ptn.match(f)
if not ff is None:
tp = ff.group().split('.')[0]
self.rule_key_value(tp)
sleep(timeout)
def async_plugin(self):
try:
app_run()
except Exception as e:
Loger().WARNING("ScoutHttpApi Exception: %s" % (e))
pass
def async_rule(self):
try:
Rule().LOOP()
except Exception as e:
Loger().WARNING("Scout rule Exception: %s" % (e))
pass
def async_dump(self):
try:
Pcapy().LOOP()
except Exception as e:
Loger().WARNING("Scout dump Exception: %s" % (e))
raise
def async_dstat(self):
try:
Dstat().LOOP()
except Exception as e:
Loger().WARNING("Scout dstat Exception: %s" % (e))
pass
