def init_accesslog1015(filename=AccessLogDir):
res, line = InitLogs(filename=filename, AccessLogRecode=True).get_access_logs3()
logging.info("1.1：入库访问日志记录【" + filename + "】第 " + str(line) + " 行")
put_opt(len(res), line=line) # 存储最新读取文本行记录的位置
# print(res)
# MPConn.insert_data(AccessLogSaveTableName, res)
MPConn.insert_data_uniq(AccessLogSaveTableName, res, key="request_id")
def kacfun(acceelog_filepath=AccessLogDir, modseclog_filepath=ModSecLogSaveTableName):
logging.info("1.0：【Start】两种日志直接入Mongo库")
init_accesslog(filename=acceelog_filepath)
init_auditlog(filename=modseclog_filepath)
logging.info("1.3：【End】两种日志直接入Mongo库")
## 开始对告警日志进行细化
from opt.detailedlog.accesslog_detailed import detailed_work
detailed_work()
## 执行存入Mysql的记录
from opt.detailedlog.mlog_to_sql import log_to_sql
log_to_sql()
def get_modseclog(self):
logging.info("2.1：执行写入告警日志详细概况")
if self.save:
## 保存只跟这个有关
from opt.wt_parse import init_auditlog_detailed, remove_alldata_modsecdetailed
# remove_alldata_modsecdetailed() # 开始有问题的时候执行
modseclog_detailed = init_auditlog_detailed()
if modseclog_detailed:
MPConn.db[ModSecLogSaveTableName +
"_detailed"].insert(modseclog_detailed)
if self.show:
for x in MPConn.db[ModSecLogSaveTableName +
"_detailed"].find(projection={"_id": False}):
print(x)
def modseclog_to_sql(self):
from utils.django_module import django_setup
django_setup()
from datetime import datetime
default_time = datetime(1900, 8, 14)
from phaser1.models import ModsecLogDetail, ModSecLogPhaserHinfo
write_counts = 0 ## 记录写入的数值
for x in self.db[ModSecLogSaveTableName +
"_detailed"].find(projection={"_id": False}):
if len(ModsecLogDetail.objects.filter(
audit_logid=x["audit_logid"])) > 0:
continue
item = ModsecLogDetail()
item.logsize = int(x["logsize"]) if "logsize" in x.keys() else 0
item.audit_logid = x["audit_logid"] if "audit_logid" in x.keys(
) else ""
item.http_user_agent = x[
"http_user_agent"] if "http_user_agent" in x.keys() else ""
item.http_ver = x["http_ver"] if "http_ver" in x.keys() else ""
item.src_host = x["src_host"] if "src_host" in x.keys() else ""
item.src_ip = x["src_ip"] if "src_ip" in x.keys() else ""
item.waf_serv = x["waf_serv"] if "waf_serv" in x.keys() else ""
item.audit_time = x["audit_time"] if "audit_time" in x.keys(
) else default_time
item.content_length = int(
x["content_length"]) if "content_length" in x.keys() else 0
item.resp_code = int(
x["resp_code"]) if "resp_code" in x.keys() else 0
item.uniq_id = x["uniq_id"] if "uniq_id" in x.keys() else ""
item.request_url = x[
"request_url"][:253] if "request_url" in x.keys() else ""
item.request_method = x[
"request_method"] if "request_method" in x.keys() else ""
item.content_type = x["content_type"] if "content_type" in x.keys(
) else ""
item.save()
if "hloginfo" in x.keys():
for y in x["hloginfo"]:
hlogitem = ModSecLogPhaserHinfo.objects.create(**dict(
rule_id=int(y["rule_id"]) if "rule_id" in
y.keys() else 0,
matched_data=y["matched_data"] if "matched_data" in
y.keys() else "",
msg=y["msg"][:225] if "msg" in y.keys() else ""))
item.hloginfo.add(hlogitem)
write_counts += 1
logging.info("3.1-写入【" + str(write_counts) + "】条告警日志进入Mysql成功")
def get_accesslog(self):
accesslogs = []
for item in MPConn.db[AccessLogSaveTableName].find(
projection={"_id": False}):
temp_dict = item.copy()
http_user_agent = item["http_user_agent"]
http_user_agent_detailed_info = get_ua_and_os_from_User_Agent(
http_user_agent)
new_item = dict(temp_dict, **http_user_agent_detailed_info)
accesslogs.append(new_item)
logging.info("执行写入访问日志详细概况")
if self.save:
MPConn.db[AccessLogSaveTableName + "_detailed"].insert(accesslogs)
if self.show:
for x in MPConn.db[AccessLogSaveTableName +
"_detailed"].find(projection={"_id": False}):
print(x)
def modseclog_to_sql(self):
from utils.django_module import django_setup
django_setup()
from datetime import datetime
default_time = datetime(1995, 8, 14)
from phaser1.models import ModsecLogDetail, ModSecLogPhaserHinfo
have_saved_audit_logid = [x.audit_logid for x in ModsecLogDetail.objects.all()]
write_counts = 0 ## 记录写入的数值
for x in self.get_latest_modseclog():
## 新增对象不可能存在重复; 所以把去重判别放在上面了
if x["audit_logid"] in have_saved_audit_logid:
continue
item = ModsecLogDetail()
item.logsize = int(x["logsize"]) if "logsize" in x.keys() else 0
item.audit_logid = x["audit_logid"] if "audit_logid" in x.keys() else ""
item.http_user_agent = x["http_user_agent"] if "http_user_agent" in x.keys() else ""
item.http_ver = x["http_ver"] if "http_ver" in x.keys() else ""
item.src_host = x["src_host"] if "src_host" in x.keys() else ""
item.src_ip = x["src_ip"] if "src_ip" in x.keys() else ""
item.server_port = x["server_port"] if "server_port" in x.keys() else "" # 2018-11-16 add port
item.waf_serv = x["waf_serv"] if "waf_serv" in x.keys() else ""
item.audit_time = x["audit_time"] if "audit_time" in x.keys() else default_time
item.content_length = int(x["content_length"]) if "content_length" in x.keys() else 0
item.resp_code = int(x["resp_code"]) if "resp_code" in x.keys() else 0
item.uniq_id = x["uniq_id"] if "uniq_id" in x.keys() else ""
item.request_url = x["request_url"][:253] if "request_url" in x.keys() else ""
item.request_method = x["request_method"] if "request_method" in x.keys() else ""
item.content_type = x["content_type"] if "content_type" in x.keys() else ""
item.server_port = x["server_port"] if "server_port" in x.keys() else self.server_port
try:
item.save()
if "hloginfo" in x.keys():
for y in x["hloginfo"]:
hlogitem = ModSecLogPhaserHinfo.objects.create(**dict(
rule_id=int(y["rule_id"]) if "rule_id" in y.keys() else 0,
matched_data=y["matched_data"] if "matched_data" in y.keys() else "",
msg=y["msg"][:225] if "msg" in y.keys() else ""
))
item.hloginfo.add(hlogitem)
write_counts += 1
except:
logging.info("该对象已存在")
logging.info("写入【"+str(write_counts)+"】条告警日志进入Mysql成功")
def log_as1():
from syslog.main import init_accesslog
from opt.detailedlog.mlog_to_sql import LogToSql
# logging.warn("=========日志导入和初始化阶段开始========")
logging.info("#访问日志#【Start】 入Mongo库")
init_accesslog()
logging.info("#访问日志#【END】 入Mongo库")
logging.info("#访问日志#【Start】 入MySql库")
LogToSql().accesslog_to_sql()
logging.info("#访问日志#【END】 入MySql库")
def put_opt(counts,
line=0,
filename=AccessLogDir,
collection_name=AccessLogSaveTableName):
## 插入记录
params = dict(
time=datetime.now(), ## 这个地方有缺陷,
# 应该再增加一个日期进来。date
filepath=filename,
line=line,
collection_name=collection_name,
counts=counts)
content = "在【"+str(params["time"])+"】读取"+filename+\
"文件`向"+ collection_name +"`中插入了[ " + str(counts) + " ]条目"
logging.info(content)
params.setdefault("content", content)
collection = MongoConn(SysLogMongoDBConfig).db[OpreationLogCollectionName]
collection.insert(params)
def accesslog_to_sql(self):
self.django_setup()
from phaser1.models import NginxAccessLogDetail
nad_datas = []
have_saved_reqids = [nad.request_id for nad in NginxAccessLogDetail.objects.all()]
for x in [y for y in self.get_latest_accsslog() if y["request_id"] \
not in have_saved_reqids]:
obj = x.copy()
try:
obj["time_local"] = get_pydt_based_logdt(re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
except:
# print(re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
logging.warn("Error:存在AccessLog日志不一样的正则 " + obj["time_local"])
continue
obj["server_port"] = self.server_port if "server_port" not in obj.keys() else 443
nad_datas.append(obj)
seccess_insert_num = self.many_insert2_accesslog(nad_datas)
logging.info("插入【" + str(seccess_insert_num) +"】条新数据到访问日志SQL数据库成功")
def log_ss2():
from syslog.main import init_auditlog
from opt.detailedlog.accesslog_detailed import detailed_work
from opt.detailedlog.mlog_to_sql import LogToSql
# logging.warn("=========日志导入和初始化阶段开始========")
# logging.info("#访问日志#【Start】 入Mongo库")
# init_accesslog()
# logging.info("#访问日志#【END】 入Mongo库")
# logging.info("#访问日志#【Start】 入MySql库")
# LogToSql().accesslog_to_sql()
# logging.info("#访问日志#【END】 入MySql库")
# logging.info("#告警日志#【Start】 入Mongo库")
logging.info("#告警日志#【Start】 入Mongo库")
init_auditlog()
logging.info("#告警日志#【Middle】重写详情入Mongo库")
detailed_work()
logging.info("#告警日志#【END】 入Mongo库")
logging.info("#告警日志#【END】 入MySql库")
LogToSql().modseclog_to_sql()
logging.info("#告警日志#【END】 入MySql库")
def work(Debug=False):
if Debug:
from syslog.init import initial_all_collections
initial_all_collections()
logging.warn("=========日志导入和初始化阶段开始========")
logging.info("1.0：【Start】两种日志直接入Mongo库")
init_accesslog()
init_auditlog()
logging.info("1.3：【End】两种日志直接入Mongo库")
## 开始对告警日志进行细化
from opt.detailedlog.accesslog_detailed import detailed_work
detailed_work()
## 执行存入Mysql的记录
from opt.detailedlog.mlog_to_sql import LogToSql
LogToSql().accesslog_to_sql()
LogToSql().modseclog_to_sql()
logging.warn("=========日志导入结束========")
def initial_all_collections(Sec=False, Ace=False):
if Ace:
db[AccessLogSaveTableName].remove()
## 实际上下面两个表在 2018-10-15 后已经失去了作用
db[CentureAccessLogManager].remove()
db[OpreationLogCollectionName].remove()
if Sec:
db[ModSecLogSaveTableName].remove()
db[ModSecLogSaveTableName + "_detailed"].remove()
from utils.django_module import django_setup
django_setup()
from phaser1.models import NginxAccessLogDetail, ModsecLogDetail, ModSecLogPhaserHinfo
if Ace:
NginxAccessLogDetail.objects.all().delete()
if Sec:
ModsecLogDetail.objects.all().delete()
ModSecLogPhaserHinfo.objects.all().delete()
logging.info("0：初始化所有的日志集合成功！")
def accesslog_to_sql(self):
from utils.django_module import django_setup
django_setup()
na_lists = []
from phaser1.models import NginxAccessLogDetail
# NginxAccessLogDetail.objects.all().delete()
from datetime import datetime
from wafmanage.utils.db_utils import from_sql_get_data
today_date = str(datetime.now().date())
try:
# query_sql = "select request_id from accesslog where date(time_local) = '{today_date}'".format(today_date=today_date)
query_sql = "select request_id from accesslog;"
have_into_mysql_ids = [
x["request_id"] for x in from_sql_get_data(query_sql)["data"]
]
# print(from_sql_get_data(query_sql)["data"] )
# print(have_into_mysql_ids)
except:
have_into_mysql_ids = []
for x in self.db[AccessLogSaveTableName].find():
obj = x.copy()
del obj["_id"]
if obj["request_id"] in have_into_mysql_ids:
continue
try:
obj["time_local"] = get_pydt_based_logdt(
re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
except:
# print(re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
logging.warn("Error:存在AccessLog日志不一样的正则 " + obj["time_local"])
return
## 记录这些条目已经存储进了 Mysql
na_lists.append(NginxAccessLogDetail(**obj))
if na_lists:
NginxAccessLogDetail.objects.bulk_create(na_lists)
logging.info("3.0: 写入【" + str(len(na_lists)) + "】条访问日志到MYSQL数据库")
def init_accesslog(filename=AccessLogDir):
res, line = InitLogs(filename=filename, AccessLogRecode=True).get_access_logs()
logging.info("1.1：入库访问日志记录【" + filename + "】第 " + str(line) + " 行")
## AccessLog 的日志最终存储到Mysql的过程中再去重
put_opt(len(res), line=line) # 存储最新读取文本行记录的位置
MPConn.insert_data(AccessLogSaveTableName, res)
def init_auditlog(filename=ModsecLogDir):
# 这里的 AccessLogRecode 是启用多个文件扫描
res = InitLogs(filename=filename, AccessLogRecode=True).get_auditlogs()
# 无重复插入
MPConn.insert_data_uniq(ModSecLogSaveTableName, res)
logging.info("1.2：入库告警日志")
def detailed_work():
logging.info("2.0：执行写入详细日志脚本开始")
# GetDetailedLog(save=True, show=False).get_accesslog()
GetDetailedLog(save=True, show=False).get_modseclog()
logging.info("2.2：执行写入详细日志脚本成功")