def main():
# 配置文件
#configFile = sys_config.getDir() + "/mdstack/conf/mdstack.conf"
configFile = os.path.split(
os.path.realpath(__file__))[0] + "/mdstack/conf/mdstack.conf"
if os.path.exists(configFile) == False:
configFile = "/opt/mdstack/conf/mdstackd/mdstack.conf"
serviceName = "datastats"
confZone = "datastats"
conf = sys_config.SysConfig(configFile)
# 进程号文件名
pidFile = conf.getConfig(confZone, "pidFile")
# 日志文件
logFile = conf.getConfig(confZone, "logFile")
# 实例名
instance = conf.getConfig(confZone, "instanceName")
daemon1 = daemon.Daemon(serviceName, pidFile, logFile, instance)
try:
lstPar, lstErrPar = getopt(
sys.argv[1:], "",
["start", "autostart", "stop", "restart", "status", "help"])
if len(lstErrPar) > 0:
print "Unknown command, try \"python run_datastats.py --help\" for more information."
elif len(lstPar) <> 1:
print "Unknown command, try \"python run_datastats.py --help\" for more information."
else:
if lstPar[0][0] == "--start":
sys_log.SysLog(logFile, instance).writeLog(
"info", "The Data Statistics Service is starting ...")
daemon1.start()
elif lstPar[0][0] == "--autostart":
sys_log.SysLog(logFile, instance).writeLog(
"info", "The Data Statistics Service is starting ...")
daemon1.start()
elif lstPar[0][0] == "--stop":
daemon1.stop()
sys_log.SysLog(logFile, instance).writeLog(
"info", "The Data Statistics Service is stopped")
elif lstPar[0][0] == "--restart":
sys_log.SysLog(logFile, instance).writeLog(
"info", "The Data Statistics Service is restarting ...")
daemon1.restart()
elif lstPar[0][0] == "--status":
daemon1.status()
elif lstPar[0][0] == "--help":
print "Usage: python run_datastats.py {Options}"
print "Options:"
print " --start: Start the Data Statistics Service"
print " --stop: Stop the Data Statistics Service"
print " --restart: Restart the Data Statistics Service"
print " --status: Show the status of the Data Statistics Service"
print " --help: Show the help"
except GetoptError, e:
print e.msg + ", try \"run_datastats.py --help\" for more information."
def hostnameToNodeids(self, hostname):
"""
通过本机的主机名获取结点id列表
"""
nodes = []
try:
# MatchQuery
# MatchQuery 替代了 TextQuery
query = pyes.query.MatchQuery("hostname", hostname)
# 连接ES
es = pyes.ES(self._url)
search = pyes.query.Search(query=query,
start=0,
size=100,
fields=[])
results = es.search(search, indices="stats", doc_types="stat")
if results != None:
for r in results.hits:
nodes.append(r._id)
except Exception, e:
sys_log.SysLog(self._logFile, self._instance).writeLog(
"error", str(traceback.format_exc()))
def chkDoit(self, lstHost, pdir, minFreeSpaceG):
"""
删除最早的pcap文件，保障磁盘空间满足最小要求
"""
try:
mst0 = pyes.query.TermsQuery(field="node", value=lstHost)
if pdir[len(pdir) - 1] == "/":
dname = pdir + "*"
else:
dname = pdir + "/*"
wild = pyes.query.WildcardQuery(field="name", value=dname)
mst1 = pyes.query.BoolQuery(should=wild)
mst_not = pyes.query.TermQuery(field="locked", value=1)
query = pyes.query.BoolQuery(must_not=mst_not)
query.add_must(mst0)
query.add_must(mst1)
search = pyes.query.Search( query=query, start=0, size=20, \
sort=[{"first": "asc"}], _source=["num", "name", "first", "size", "node"] )
# 连接ES
es = pyes.ES(self._url)
results = es.search(search, indices="files", doc_types="file")
if results != None:
total = results.total
if total <= 20:
return
for r in results.hits:
if os.path.exists(r._source["name"]) == False:
# 删除files索引中的记录
es.delete("files", "file", r._id)
total = total - 1
if total <= 20:
break
else:
freeG = self.getFreeDiskSpace(r._source["name"])
if (freeG < minFreeSpaceG):
# 删除文件
os.remove(r._source["name"])
# 删除files索引中的记录
es.delete("files", "file", r._id)
total = total - 1
if total <= 20:
break
except Exception, e:
print e
sys_log.SysLog(self._logFile, self._instance).writeLog(
"error", str(traceback.format_exc()))
def getFreeDiskSpace(self, pcapDir):
"""
获取文件所在路径的空余空间, 单位为G
"""
fds = 0
try:
disk = os.statvfs(pcapDir)
fds = disk.f_frsize / 1024.0 * disk.f_bavail / (1024.0 * 1024.0)
except Exception, e:
print e
sys_log.SysLog(self._logFile, self._instance).writeLog(
"error", str(traceback.format_exc()))
return None
def stats_firewall_dashboard():
"""
自定义防火墙dashboard，统一调用入口
"""
# 配置文件
configFile = "/opt/mdstack/conf/mdstackd/mdstack.conf"
conf = sys_config.SysConfig(configFile)
logFile = conf.getConfig("datastats", "logFile")
instance = conf.getConfig("datastats", "instanceName")
es_host = conf.getConfig("elasticsearch", "esHost")
es_port = conf.getConfig("elasticsearch", "esPort")
post_host = conf.getConfig("postgresql", "dbHost")
post_port = conf.getConfig("postgresql", "dbPort")
post_user = conf.getConfig("postgresql", "dbUser")
post_pwd = conf.getConfig("postgresql", "dbPwd")
post_db = conf.getConfig("postgresql", "dbName")
# 删除过期的统计数据
post_url = "host=" + post_host + " port=" + post_port + " user="******" password="******" dbname=" + post_db
days = post_data.get_expiry_date(post_url, "flows") + 1
idx_name = "firewall_dashboard"
url = 'http://' + es_host + ":" + es_port
print url
conn = ES(url, timeout=120)
localtz = timezone('Asia/Shanghai')
# 基础查询条件，这里一般设置查询防火墙的条件
base_condition = "(msg.dn:\"hw:venustech:usg_fw_3610d\")"
# 统计各台防火墙在每个小时内被pass/deny的访问次数、字节（发送/接收/总数）、数据包（发送/接收/总数）
if conf.getConfig("custom_dashboard", "firewall.fw_stats_per_hour") == "Y":
try:
doc_type = "fw_stats_per_hour"
# 删除过期的统计数据
clear_index(idx_name, doc_type, conn, days)
day = datetime.now() - timedelta(1)
fw_stats_per_hour(idx_name, doc_type, conn, day, localtz,
base_condition)
day = datetime.now() - timedelta(2)
fw_stats_per_hour(idx_name, doc_type, conn, day, localtz,
base_condition)
except Exception, e:
sys_log.SysLog(logFile,
instance).writeLog("error",
str(traceback.format_exc()))
def stats_flows_dashboard():
"""
自定义流量dashboard，统一调用入口
"""
# 配置文件
configFile = "/opt/mdstack/conf/mdstackd/mdstack.conf"
conf = sys_config.SysConfig(configFile)
logFile = conf.getConfig("datastats", "logFile")
instance = conf.getConfig("datastats", "instanceName")
es_host = conf.getConfig("elasticsearch", "esHost")
es_port = conf.getConfig("elasticsearch", "esPort")
post_host = conf.getConfig("postgresql", "dbHost")
post_port = conf.getConfig("postgresql", "dbPort")
post_user = conf.getConfig("postgresql", "dbUser")
post_pwd = conf.getConfig("postgresql", "dbPwd")
post_db = conf.getConfig("postgresql", "dbName")
# 删除过期的统计数据
post_url = "host=" + post_host + " port=" + post_port + " user="******" password="******" dbname=" + post_db
days = post_data.get_expiry_date(post_url, "flows") + 1
idx_name = "flows_dashboard"
url = 'http://' + es_host + ":" + es_port
print url
conn = ES(url, timeout=120)
localtz = timezone('Asia/Shanghai')
# 基础查询条件，这里一般设置通用查询的条件
base_condition = ""
# 统计昨天、前天每个小时内网络流量的session次数、包数、字节数、数据字节数、时延
if conf.getConfig("custom_dashboard", "flows.flows_stats_per_hour") == "Y":
try:
doc_type = "flows_stats_per_hour"
# 删除过期的统计数据
clear_index(idx_name, doc_type, conn, days)
day = datetime.now() - timedelta(1)
flows_stats_per_hour(idx_name, doc_type, conn, day, localtz,
base_condition)
day = datetime.now() - timedelta(2)
flows_stats_per_hour(idx_name, doc_type, conn, day, localtz,
base_condition)
except Exception, e:
sys_log.SysLog(logFile,
instance).writeLog("error",
str(traceback.format_exc()))
def stats_dashboard(pd):
"""
统一调用入口函数
"""
try:
if pd != None and pd.is_master() == False:
return
# 自定义防火墙dashboard统计
firewall_dashboard.stats_firewall_dashboard()
flows_dashboard.stats_flows_dashboard()
except Exception, e:
# 配置文件
configFile = "/opt/mdstack/conf/mdstackd/mdstack.conf"
conf = sys_config.SysConfig(configFile)
# 错误日志
logFile = conf.getConfig("datastats", "logFile")
instance = conf.getConfig("datastats", "instanceName")
sys_log.SysLog(logFile, instance).writeLog("error",
str(traceback.format_exc()))
# 删除过期的统计数据
clear_index(idx_name, doc_type, conn, days)
# 分组的字段
field = "gw_proto_port"
# 取topN
size = 100
day = datetime.now() - timedelta(1)
fw_top_count_per_day(idx_name, doc_type, conn, day, localtz, field,
size, base_condition)
day = datetime.now() - timedelta(2)
fw_top_count_per_day(idx_name, doc_type, conn, day, localtz, field,
size, base_condition)
except Exception, e:
sys_log.SysLog(logFile,
instance).writeLog("error",
str(traceback.format_exc()))
# 统计"源IP"在每天内被pass/deny的次数的TOPN，并统计TOPN中的各"源IP"的24小时分时次数
if conf.getConfig("custom_dashboard",
"firewall.fw_srcip_count_per_day") == "Y":
try:
doc_type = "fw_srcip_count_per_day"
# 删除过期的统计数据
clear_index(idx_name, doc_type, conn, days)
# 分组的字段
field = "gw_src_ipv4"
# 取topN
size = 100
day = datetime.now() - timedelta(1)