class Invasion(object):
def __init__(self):
self.mongo = MongoHelper(MongoConfig.uri, db=MongoConfig.db, collection=MongoConfig.invasions_collection)
def new(self, *activities) -> ObjectId:
"""
新建入侵事件 返回入侵事件ID
"""
source_ip = activities[0]["form_data"]["source_ip"]
title = "来自于{ip}的入侵事件".format(ip=source_ip)
doc = {
"title": title,
"level": _get_max_level(list(map(lambda x: x["level"], activities))),
"start_time": activities[0]["start_time"],
"end_time": activities[-1]["end_time"],
"source_ip": source_ip,
"status": "pending"
}
return self.mongo.insert_one(doc).inserted_id
def add_activity(self, invasion_id: ObjectId, activity_doc: dict):
"""
向当前入侵事件添加一条新的威胁活动
"""
query = {
"_id": invasion_id
}
invasion = self.mongo.find_one(query=query)
level = _get_max_level([invasion["level"], activity_doc["level"]])
end_time = activity_doc["end_time"]
if end_time > invasion["end_time"]:
self.update(invasion_id, {
"$set": {
"level": level,
"end_time": end_time
}
})
else:
self.update(invasion_id, {
"$set": {
"level": level
}
})
def find_record(self, source_ip, start_time, **kwargs):
"""
根据 source_ip 查找一段时间内的相同来源的入侵事件
"""
return self.mongo.find_one({
"source_ip": source_ip,
"end_time": {"$gte": start_time + timedelta(hours=-main_config.merge_invasion_time)},
**kwargs
})
def update(self, _id, doc):
self.mongo.update_one({
"_id": _id
}, doc=doc)
def init_sensitive_groups(domain):
logger.info("init sensitive groups.")
domain = get_netbios_domain(domain)
ldap_search = LDAPSearch(domain)
redis = RedisHelper()
mongo = MongoHelper(uri=MongoConfig.uri,
db=MongoConfig.db,
collection=MongoConfig.settings_collection)
sensitive_groups = []
for item in default_sensitive_groups(domain):
if len(item["sid"]) > 0:
sensitive_groups.append(item)
else:
entry = ldap_search.search_by_name(item["name"],
attributes=["objectSid"])
if not entry or len(
entry.entry_attributes_as_dict["objectSid"]) == 0:
continue
sid = entry.entry_attributes_as_dict["objectSid"][0]
item["sid"] = sid
sensitive_groups.append(item)
logger.info(",".join(list(map(lambda x: x["name"], sensitive_groups))))
sensitive_entry = mongo.find_one({"name": "sensitive_entry"})["value"]
sensitive_entry["group"] = sensitive_groups
mongo.update_one({"name": "sensitive_entry"},
{"$set": {
"value": sensitive_entry
}},
upsert=True)
redis.set_str_value("sensitive_entry" + REDIS_KEY_SUFFIX,
simplejson.dumps(sensitive_entry))
def set_learning_end_time_setting():
value = move_n_days(datetime_utc_now_obj(), 10)
logger.info("set learning end time: " + str(value))
name = "learning_end_time"
redis = RedisHelper()
mongo = MongoHelper(uri=MongoConfig.uri,
db=MongoConfig.db,
collection=MongoConfig.settings_collection)
mongo.update_one(filter={"name": name},
doc={"$set": {
"value": value
}},
upsert=True)
key = name + REDIS_KEY_SUFFIX
redis.set_str_value(key, datetime_to_common_str(value))
class Delegation(object):
def __init__(self):
self.mongo = MongoHelper(MongoConfig.uri, MongoConfig.db,
MongoConfig.delegation_collection)
def new_delegation_record(self,
user: User,
delegation_type: str,
allowed_to=None):
if delegation_type not in [
CONSTRAINED_DELEGATION, UNCONSTRAINED_DELEGATION,
RES_BASED_CONSTRAINED_DELEGATION
]:
raise NoSuchDelegationType()
self.mongo.insert_one({
"name": user.user_name,
"sid": user.user_sid,
"domain": user.domain_name,
"delegation_type": delegation_type,
"allowed_to": allowed_to
})
def find_constrained_delegation_by_sid(self, sid: str):
return self.mongo.find_one({
"sid": sid,
"delegation_type": CONSTRAINED_DELEGATION
})
def find_res_constrained_delegation_by_name(self, name: str):
return self.mongo.find_one({
"name":
name,
"delegation_type":
RES_BASED_CONSTRAINED_DELEGATION
})
def find_one_delegation(self, query: dict):
return self.mongo.find_one(query)
def update_delegation(self, sid, delegation_type, allowed_to):
self.mongo.update_one({
"sid": sid,
"delegation_type": delegation_type
}, {"allowed_to": allowed_to})
def init_ldap_settings(domain, server, user, password):
logger.info("init the ldap configuration.")
if not server.startswith("ldap://"):
server = "ldap://" + server
mongo = MongoHelper(uri=MongoConfig.uri,
db=MongoConfig.db,
collection=MongoConfig.settings_collection)
query = {"name": "ldap"}
doc = {
domain: {
"server": server,
"user": user,
"password": password,
"dn": get_dn_domain_name(domain)
}
}
mongo.update_one(filter=query, doc={"$set": {"value": doc}}, upsert=True)
redis = RedisHelper()
redis.set_str_value("ldap" + REDIS_KEY_SUFFIX, simplejson.dumps(doc))
def init_default_settings(domain):
logger.info("init other settings.")
redis = RedisHelper()
mongo = MongoHelper(uri=MongoConfig.uri,
db=MongoConfig.db,
collection=MongoConfig.settings_collection)
for name, value in default_settings.items():
if name == "domain_list":
value = [domain]
mongo.update_one(filter={"name": name},
doc={"$set": {
"value": value
}},
upsert=True)
key = name + REDIS_KEY_SUFFIX
if name in [
"domain_list", "VPN_ip_part",
"detail_file_share_white_list_setting"
]:
if len(value) != 0:
redis.set_list(key, *value)
elif name in [
"raw_data_expire", "honeypot_account", "alarms_merge",
"sensitive_entry", "kerberos"
]:
redis.set_str_value(key, simplejson.dumps(value))
elif name in ["brute_force_max"]:
redis.set_str_value(key, str(value))
elif isinstance(value, list):
if len(value) > 0 and isinstance(value[0], dict):
redis.set_str_value(key, simplejson.dumps(value))
else:
if len(value) != 0:
redis.set_list(key, *value)
elif isinstance(value, str):
redis.set_str_value(key, value)
elif isinstance(value, dict):
redis.set_str_value(key, simplejson.dumps(value))
elif isinstance(value, int):
redis.set_str_value(key, str(value))
def get_all_dc_names(domain: str):
"""
将DC列表入库
"""
domain = get_netbios_domain(domain)
logger.info("Search all domain controllers using LDAP.")
dc_name_list = []
ldap_search = LDAPSearch(domain)
dc_list = ldap_search.search_domain_controller()
for each in dc_list:
dc_name = str(each["cn"])
dc_name_list.append(dc_name)
mongo = MongoHelper(MongoConfig.uri, MongoConfig.db,
MongoConfig.settings_collection)
doc = {domain: dc_name_list}
logger.info(",".join(dc_name_list))
logger.info(
"domain controller count: {count}".format(count=len(dc_name_list)))
logger.info("Save all domain controllers to settings.")
mongo.update_one({"name": "dc_name_list"}, {"$set": {"value": doc}}, True)
redis = RedisHelper()
redis.set_str_value("dc_name_list" + REDIS_KEY_SUFFIX,
simplejson.dumps(doc))
class Alert(object):
def __init__(self):
self.alert_mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.alerts_collection)
self.activity = Activity()
self.ignore_rule = MatchRules(MongoConfig.ignore_collection)
self.exclude_rule = MatchRules(MongoConfig.exclude_collection)
def generate(self, doc):
"""
生成告警，发送邮件并入库
"""
# 计算告警表单内容的唯一ID
form_data_id = _get_form_data_md5(doc["form_data"])
doc["form_data_id"] = form_data_id
# 误报排除规则过滤，不再产生记录
if self._auto_exclude(doc):
return
# 忽略规则过滤
doc = self._auto_ignore(doc)
# 首先尝试合并相同来源且相同类型的告警到同一个威胁活动， 即unique_id重复的告警
if self._merge_alert(doc):
return
# 新增，生成威胁活动
activity_id = self.activity.new(doc)
# 记录下威胁活动的ID以后，告警入库
doc["activity_id"] = activity_id
self.alert_mongo.insert_one(doc)
def _merge_alert(self, alert_doc):
"""
合并告警到同一个威胁活动
"""
# 查找是否已经生成了威胁活动，没有的话直接退出，然后新增
activity = self.activity.find_record(alert_doc["unique_id"], alert_doc["start_time"])
if not activity:
return False
alert_doc["activity_id"] = activity["_id"]
# 尝试合并告警表单内容完全重复的告警，增加重复次数
if self._merge_repeat_count(alert_doc):
self.activity.update(activity["_id"], {
"$set": {
"end_time": alert_doc["end_time"]
}
})
# 无完全重复，在该威胁活动下新增一条告警
else:
self.activity.add_alert(activity, alert_doc)
self.alert_mongo.insert_one(alert_doc)
return True
def _merge_repeat_count(self, alert_doc: dict) -> bool:
"""
完全重复的告警内容 不再入库 直接统计次数
"""
query = {
"activity_id": alert_doc["activity_id"],
"form_data_id": alert_doc["form_data_id"]
}
record = self.alert_mongo.find_one(query)
if record:
# 账号爆破特殊处理一下
if alert_doc["alert_code"] == "301":
self.alert_mongo.update_one(
{"_id": record["_id"]},
{
"$set": {
"form_data.brute_force_target_users": alert_doc["form_data"]["brute_force_target_users"]
}
}
)
return True
self.alert_mongo.update_one(
{"_id": record["_id"]},
{
"$inc": {"repeat_count": 1},
"$set": {"end_time": alert_doc["end_time"]}
}
)
return True
else:
return False
def _auto_exclude(self, doc):
"""
根据预先设定的规则，自动排除误报
和忽略的区别是，排除的误报不再产生记录，直接忽略
"""
rule_id = self.exclude_rule.match(doc)
if rule_id:
return True
else:
return False
def _auto_ignore(self, doc):
"""
根据预先设定的规则，自动忽略告警
忽略的告警也需要合并
"""
rule_id = self.ignore_rule.match(doc)
if rule_id:
doc["status"] = "auto_ignore"
doc["ignore_rule_id"] = rule_id
return doc
class Activity(object):
def __init__(self):
self.activity_mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.activities_collection)
self.invasion = Invasion()
def new(self, activity_doc: dict) -> ObjectId:
"""
新增威胁活动记录
同时尝试生成入侵事件记录
"""
invasion_id = self._generate_invasion(activity_doc)
if invasion_id:
activity_doc["invasion_id"] = invasion_id
return self.activity_mongo.insert_one(activity_doc).inserted_id
def add_alert(self, activity_doc, alert_doc: dict):
doc = {
"$set": {}
}
if alert_doc["end_time"] > activity_doc["end_time"]:
doc["$set"]["end_time"] = alert_doc["end_time"]
if alert_doc["level"] > activity_doc["level"]:
doc["$set"]["level"] = alert_doc["level"]
if len(doc["$set"].keys()) == 0:
return
self.activity_mongo.update_one({
"_id": activity_doc["_id"]
}, doc)
def update(self, _id, doc):
self.activity_mongo.update_one({
"_id": _id
}, doc=doc)
def find_record(self, uid, start_time):
"""
根据 unique_id 查找一段时间内相同的威胁活动
"""
return self.activity_mongo.find_one({
"unique_id": uid,
"end_time": {"$gte": start_time + timedelta(hours=-main_config.merge_activity_time)}
})
def _generate_invasion(self, activity_doc) -> ObjectId:
"""
生成入侵事件
多个相同来源IP 不同类型的威胁活动 可以生成一个入侵事件
返回入侵事件的ID
"""
# 首先查找是否已存在对应的入侵事件，尝试合并
invasion_id = self._merge_activity(activity_doc)
if invasion_id:
assert isinstance(invasion_id, ObjectId)
return invasion_id
# 没有已存在的入侵事件，则按照条件，查找是否已存在不同类型的威胁活动
query = {
"form_data.source_ip": activity_doc["form_data"]["source_ip"],
"end_time": {"$gte": activity_doc["start_time"] + timedelta(hours=-main_config.merge_invasion_time)},
"alert_code": {"$ne": activity_doc["alert_code"]}
}
# 如果存在不同类型的威胁活动
another_activities = list(self.activity_mongo.find_all(query).sort("start_time", ASCENDING))
if len(another_activities) > 0:
invasion_id = self.invasion.new(*another_activities, activity_doc)
# 创建完入侵事件之后，将之前查询的到威胁活动全部加上对应的ID
for activity in another_activities:
self.update(activity["_id"], {
"$set": {
"invasion_id": invasion_id
}
})
def _merge_activity(self, activity_doc):
source_ip = activity_doc["form_data"]["source_ip"]
invasion = self.invasion.find_record(source_ip, activity_doc["start_time"])
if not invasion:
return False
# 向存在的入侵事件添加当前威胁活动
self.invasion.add_activity(invasion["_id"], activity_doc)
return invasion["_id"]