def load_settings():
"""
加载Mongo中保存的配置信息到redis中
"""
mongo = MongoHelper(MongoConfig.uri, MongoConfig.db,
MongoConfig.settings_collection)
redis = RedisHelper()
fetcher = mongo.find_all({})
for each in fetcher:
key = each["name"] + "_setting"
# 再录入
if isinstance(each["value"], list):
if len(each["value"]) == 0:
continue
elif isinstance(each["value"][0], dict):
redis.set_str_value(key, simplejson.dumps(each["value"]))
continue
redis.set_list(key, *each["value"])
elif isinstance(each["value"], dict):
redis.set_str_value(key, simplejson.dumps(each["value"]))
else:
redis.set_str_value(key, each["value"])
class MatchRules(object):
def __init__(self, rules_table):
self.mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, rules_table)
def match(self, event_doc: dict):
alert_code = event_doc["alert_code"]
form_data = event_doc["form_data"]
all_rules = self.mongo.find_all({"alert_code": alert_code})
for rule in all_rules:
try:
rule = Rule(rule)
# 成功匹配规则 则返回被匹配上的规则ID
if self._match_one(rule, form_data):
return rule.id
except Exception as e:
return False
return False
def _match_one(self, rule: Rule, form_data):
"""
一条忽略内容的多条规则，全部匹配才算成功
"""
for rule_content in rule.rules:
if rule_content.field_type == "string":
if not self._match_string(form_data[rule_content.field_name],
rule_content.value,
rule_content.match_type):
return False
elif rule_content.field_type == "ip":
if not self._match_ip(form_data[rule_content.field_name],
rule_content.value):
return False
elif rule_content.field_type == "list":
if not self._match_list(form_data[rule_content.field_name],
rule_content.value,
rule_content.match_type):
return False
return True
def _match_string(self, alert_value, rule_value, match_type) -> bool:
"""
匹配字符串，将告警字段内容和忽略规则的内容匹配
"""
if match_type == "term":
if isinstance(alert_value, list):
for each in alert_value:
if each == rule_value:
return True
return False
else:
return alert_value == rule_value
# 正则匹配
elif match_type == "regex":
if isinstance(alert_value, list):
for each in alert_value:
if re.match(rule_value, each):
return True
return False
else:
return True if re.match(rule_value, alert_value) else False
def _match_ip(self, alert_ip, rule_ip) -> bool:
"""
匹配IP类型
"""
# CIDR
if "/" in rule_ip:
return IP(alert_ip) in IP(rule_ip)
else:
return alert_ip == rule_ip
def _match_list(self, alert_value, rule_value, match_type):
"""
匹配列表
"""
for each in rule_value:
if match_type == "ip" and self._match_ip(alert_value, each):
return True
elif match_type == "string" or match_type == "regex":
if self._match_string(alert_value, each, match_type):
return True
return False
class Engine(object):
def __init__(self):
self.event_log_modules_map = None
# self.traffic_kerberos_modules_map = None
self.mongo = MongoHelper(MongoConfig.uri, MongoConfig.db,
MongoConfig.delay_run_collection)
self.alert = Alert()
def load(self):
# 加载事件日志检测模块
logger.info("loading detect modules based on event_log")
self.event_log_modules_map = self._load_module("event_log", "EVENT_ID")
# 加载kerberos流量检测模块
# logger.info("loading detect modules based on traffic_kerberos")
# self.traffic_kerberos_modules_map = self._load_module("traffic_kerberos", "MSG_TYPE")
def start(self):
"""
引擎启动主入口
"""
self.load()
# 启动消费者
c = Consumer()
# 注册回调
logger.info("start MQ consumer and register callback func.")
logger.info("status: main process running")
c.run(self.do_analyze)
def delay_run(self):
"""
延迟检测
** 请单进程运行！ **
"""
self.load()
logger.info("status: delay process running")
while True:
time.sleep(5)
data_list = self._get_delay_data()
for data in data_list:
alert_code = data["_delay_info"]["alert_code"]
# if data["type"] == "krb5":
# krb = Kerberos(data)
# self._run_analyze(data=krb, data_type=krb.msg_type, modules_map=self.traffic_kerberos_modules_map,
# alert_code=alert_code)
if data["type"] == "wineventlog":
log = Log(data)
self._run_analyze(data=log,
data_type=log.event_id,
modules_map=self.event_log_modules_map,
alert_code=alert_code)
# 删除完成检测数据
self._clear_confirmed_data(data_list)
def do_analyze(self, data: dict):
# 解析krb5流量
# if data["type"] == "krb5":
# krb = Kerberos(data)
# if krb.msg_type not in self.traffic_kerberos_modules_map:
# return
# self._run_analyze(data=krb, data_type=krb.msg_type, modules_map=self.traffic_kerberos_modules_map)
# 解析事件日志
if data["type"] == "wineventlog":
if data["event_id"] == 4662:
return
if "event_data" not in data and data["event_id"] != 1100:
return
log = Log(data)
if log.event_id not in self.event_log_modules_map:
return
self._run_analyze(data=log,
data_type=log.event_id,
modules_map=self.event_log_modules_map)
def _run_analyze(self,
data,
data_type,
modules_map: dict,
alert_code=None):
"""
运行检测模块
:param data: 数据字典
:param data_type: log.event_id 的值或者 krb.msg_type
:param modules_map: 加载了检测模块的字典
:param alert_code: 可选，具体检测的告警代码，指定了之后只运行该模块
:return:
"""
module_list = modules_map[data_type]
for module in module_list:
code = module["code"]
if alert_code and alert_code != code:
continue
m_object = module["object"]
# 运行检测模块的语句
alert_doc = m_object.run(data)
if alert_doc:
# 存在问题，告警
self.alert.generate(alert_doc)
def _load_module(self, name: str, data_type: str) -> dict:
modules_map = {}
def _register_module(d_type, m):
if d_type not in modules_map:
modules_map[d_type] = [m]
else:
modules_map[d_type].append(m)
file_list = get_walk_files(project_dir + "/modules/detect/" + name)
for f in file_list:
f = f.replace(project_dir, ".")
module_path, f = format_module_path(f)
module = __import__(module_path, fromlist=[f])
logger.info("loaded module: " + module_path)
data_types = getattr(module, data_type)
assert isinstance(data_types, list)
for d_type in data_types:
_register_module(
d_type, {
"code":
getattr(module, "ALERT_CODE") if hasattr(
module, "ALERT_CODE") else None,
"object":
getattr(module, f)()
})
return modules_map
def _get_delay_data(self):
query = {"_delay_info.time": {"$lte": datetime_now_obj()}}
return [each for each in self.mongo.find_all(query)]
def _clear_confirmed_data(self, data_list):
id_list = []
for data in data_list:
id_list.append(data["_id"])
query = {"_id": {"$in": id_list}}
self.mongo.delete_many(query)
class Activity(object):
def __init__(self):
self.activity_mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.activities_collection)
self.invasion = Invasion()
def new(self, activity_doc: dict) -> ObjectId:
"""
新增威胁活动记录
同时尝试生成入侵事件记录
"""
invasion_id = self._generate_invasion(activity_doc)
if invasion_id:
activity_doc["invasion_id"] = invasion_id
return self.activity_mongo.insert_one(activity_doc).inserted_id
def add_alert(self, activity_doc, alert_doc: dict):
doc = {
"$set": {}
}
if alert_doc["end_time"] > activity_doc["end_time"]:
doc["$set"]["end_time"] = alert_doc["end_time"]
if alert_doc["level"] > activity_doc["level"]:
doc["$set"]["level"] = alert_doc["level"]
if len(doc["$set"].keys()) == 0:
return
self.activity_mongo.update_one({
"_id": activity_doc["_id"]
}, doc)
def update(self, _id, doc):
self.activity_mongo.update_one({
"_id": _id
}, doc=doc)
def find_record(self, uid, start_time):
"""
根据 unique_id 查找一段时间内相同的威胁活动
"""
return self.activity_mongo.find_one({
"unique_id": uid,
"end_time": {"$gte": start_time + timedelta(hours=-main_config.merge_activity_time)}
})
def _generate_invasion(self, activity_doc) -> ObjectId:
"""
生成入侵事件
多个相同来源IP 不同类型的威胁活动 可以生成一个入侵事件
返回入侵事件的ID
"""
# 首先查找是否已存在对应的入侵事件，尝试合并
invasion_id = self._merge_activity(activity_doc)
if invasion_id:
assert isinstance(invasion_id, ObjectId)
return invasion_id
# 没有已存在的入侵事件，则按照条件，查找是否已存在不同类型的威胁活动
query = {
"form_data.source_ip": activity_doc["form_data"]["source_ip"],
"end_time": {"$gte": activity_doc["start_time"] + timedelta(hours=-main_config.merge_invasion_time)},
"alert_code": {"$ne": activity_doc["alert_code"]}
}
# 如果存在不同类型的威胁活动
another_activities = list(self.activity_mongo.find_all(query).sort("start_time", ASCENDING))
if len(another_activities) > 0:
invasion_id = self.invasion.new(*another_activities, activity_doc)
# 创建完入侵事件之后，将之前查询的到威胁活动全部加上对应的ID
for activity in another_activities:
self.update(activity["_id"], {
"$set": {
"invasion_id": invasion_id
}
})
def _merge_activity(self, activity_doc):
source_ip = activity_doc["form_data"]["source_ip"]
invasion = self.invasion.find_record(source_ip, activity_doc["start_time"])
if not invasion:
return False
# 向存在的入侵事件添加当前威胁活动
self.invasion.add_activity(invasion["_id"], activity_doc)
return invasion["_id"]