def remove_today_writelogs():
from logConfig import logging
logging.warn("今日初始化 `accesss_log` 的所有访问日志 ")
collection = MongoConn(SysLogMongoDBConfig).db[OpreationLogCollectionName]
data = collection.find().sort([
("time", pymongo.DESCENDING),
])[0]
if str(data["time"].date()) == str(datetime.now().date()):
collection.remove({"_id": data["_id"]})
def __init__(self, filename="localhost-80.access.log", MAX_INSERT_NUM=200):
self.filename=filename
try:
self.server_port = re.match(".*\-(\d+)\.\w+\.log", str(filename).split("/")[-1]).group(1)
if sys.platform == "win32":
self.server_port = re.match(".*\-(\d+)\.\w+\.log",str(filename).split("\\")[-1]).group(1)
except:
logging.warn("SERVER_PORT_PARSE_ERROR")
self.server_port = 80
self.MAX_INSERT_NUM=MAX_INSERT_NUM
def accesslog_to_sql(self):
self.django_setup()
from phaser1.models import NginxAccessLogDetail
nad_datas = []
have_saved_reqids = [nad.request_id for nad in NginxAccessLogDetail.objects.all()]
for x in [y for y in self.get_latest_accsslog() if y["request_id"] \
not in have_saved_reqids]:
obj = x.copy()
try:
obj["time_local"] = get_pydt_based_logdt(re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
except:
# print(re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
logging.warn("Error:存在AccessLog日志不一样的正则 " + obj["time_local"])
continue
obj["server_port"] = self.server_port if "server_port" not in obj.keys() else 443
nad_datas.append(obj)
seccess_insert_num = self.many_insert2_accesslog(nad_datas)
logging.info("插入【" + str(seccess_insert_num) +"】条新数据到访问日志SQL数据库成功")
def work(Debug=False):
if Debug:
from syslog.init import initial_all_collections
initial_all_collections()
logging.warn("=========日志导入和初始化阶段开始========")
logging.info("1.0：【Start】两种日志直接入Mongo库")
init_accesslog()
init_auditlog()
logging.info("1.3：【End】两种日志直接入Mongo库")
## 开始对告警日志进行细化
from opt.detailedlog.accesslog_detailed import detailed_work
detailed_work()
## 执行存入Mysql的记录
from opt.detailedlog.mlog_to_sql import LogToSql
LogToSql().accesslog_to_sql()
LogToSql().modseclog_to_sql()
logging.warn("=========日志导入结束========")
def accesslog_to_sql(self):
from utils.django_module import django_setup
django_setup()
na_lists = []
from phaser1.models import NginxAccessLogDetail
# NginxAccessLogDetail.objects.all().delete()
from datetime import datetime
from wafmanage.utils.db_utils import from_sql_get_data
today_date = str(datetime.now().date())
try:
# query_sql = "select request_id from accesslog where date(time_local) = '{today_date}'".format(today_date=today_date)
query_sql = "select request_id from accesslog;"
have_into_mysql_ids = [
x["request_id"] for x in from_sql_get_data(query_sql)["data"]
]
# print(from_sql_get_data(query_sql)["data"] )
# print(have_into_mysql_ids)
except:
have_into_mysql_ids = []
for x in self.db[AccessLogSaveTableName].find():
obj = x.copy()
del obj["_id"]
if obj["request_id"] in have_into_mysql_ids:
continue
try:
obj["time_local"] = get_pydt_based_logdt(
re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
except:
# print(re.match("(.*?)\s(.*)", obj["time_local"]).group(1))
logging.warn("Error:存在AccessLog日志不一样的正则 " + obj["time_local"])
return
## 记录这些条目已经存储进了 Mysql
na_lists.append(NginxAccessLogDetail(**obj))
if na_lists:
NginxAccessLogDetail.objects.bulk_create(na_lists)
logging.info("3.0: 写入【" + str(len(na_lists)) + "】条访问日志到MYSQL数据库")
from fatal_v11.tests.test_common_log import test_accesslog_getting, test_auditlog_getting
from fatal_v11.tests.test_common_log_to_db import test_accesslog_2_mysql, test_modseclog_2_mysql, test_accesslog_2_mongo, test_modseclog_2_mongo
from fatal_v11.v11 import sql_log
from logConfig import logging
if __name__ == '__main__':
# test_accesslog_getting()
# test_auditlog_getting()
# test_modseclog_2_mysql()
# test_accesslog_2_mongo()
# test_modseclog_2_mongo()
sql_log()
logging.warn("【apscheduler】执行日志写入Mysql数据库成功")