class Invasion(object):
def __init__(self):
self.mongo = MongoHelper(MongoConfig.uri, db=MongoConfig.db, collection=MongoConfig.invasions_collection)
def new(self, *activities) -> ObjectId:
"""
新建入侵事件 返回入侵事件ID
"""
source_ip = activities[0]["form_data"]["source_ip"]
title = "来自于{ip}的入侵事件".format(ip=source_ip)
doc = {
"title": title,
"level": _get_max_level(list(map(lambda x: x["level"], activities))),
"start_time": activities[0]["start_time"],
"end_time": activities[-1]["end_time"],
"source_ip": source_ip,
"status": "pending"
}
return self.mongo.insert_one(doc).inserted_id
def add_activity(self, invasion_id: ObjectId, activity_doc: dict):
"""
向当前入侵事件添加一条新的威胁活动
"""
query = {
"_id": invasion_id
}
invasion = self.mongo.find_one(query=query)
level = _get_max_level([invasion["level"], activity_doc["level"]])
end_time = activity_doc["end_time"]
if end_time > invasion["end_time"]:
self.update(invasion_id, {
"$set": {
"level": level,
"end_time": end_time
}
})
else:
self.update(invasion_id, {
"$set": {
"level": level
}
})
def find_record(self, source_ip, start_time, **kwargs):
"""
根据 source_ip 查找一段时间内的相同来源的入侵事件
"""
return self.mongo.find_one({
"source_ip": source_ip,
"end_time": {"$gte": start_time + timedelta(hours=-main_config.merge_invasion_time)},
**kwargs
})
def update(self, _id, doc):
self.mongo.update_one({
"_id": _id
}, doc=doc)
class Delegation(object):
def __init__(self):
self.mongo = MongoHelper(MongoConfig.uri, MongoConfig.db,
MongoConfig.delegation_collection)
def new_delegation_record(self,
user: User,
delegation_type: str,
allowed_to=None):
if delegation_type not in [
CONSTRAINED_DELEGATION, UNCONSTRAINED_DELEGATION,
RES_BASED_CONSTRAINED_DELEGATION
]:
raise NoSuchDelegationType()
self.mongo.insert_one({
"name": user.user_name,
"sid": user.user_sid,
"domain": user.domain_name,
"delegation_type": delegation_type,
"allowed_to": allowed_to
})
def find_constrained_delegation_by_sid(self, sid: str):
return self.mongo.find_one({
"sid": sid,
"delegation_type": CONSTRAINED_DELEGATION
})
def find_res_constrained_delegation_by_name(self, name: str):
return self.mongo.find_one({
"name":
name,
"delegation_type":
RES_BASED_CONSTRAINED_DELEGATION
})
def find_one_delegation(self, query: dict):
return self.mongo.find_one(query)
def update_delegation(self, sid, delegation_type, allowed_to):
self.mongo.update_one({
"sid": sid,
"delegation_type": delegation_type
}, {"allowed_to": allowed_to})
class Alert(object):
def __init__(self):
self.alert_mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.alerts_collection)
self.activity = Activity()
self.ignore_rule = MatchRules(MongoConfig.ignore_collection)
self.exclude_rule = MatchRules(MongoConfig.exclude_collection)
def generate(self, doc):
"""
生成告警，发送邮件并入库
"""
# 计算告警表单内容的唯一ID
form_data_id = _get_form_data_md5(doc["form_data"])
doc["form_data_id"] = form_data_id
# 误报排除规则过滤，不再产生记录
if self._auto_exclude(doc):
return
# 忽略规则过滤
doc = self._auto_ignore(doc)
# 首先尝试合并相同来源且相同类型的告警到同一个威胁活动， 即unique_id重复的告警
if self._merge_alert(doc):
return
# 新增，生成威胁活动
activity_id = self.activity.new(doc)
# 记录下威胁活动的ID以后，告警入库
doc["activity_id"] = activity_id
self.alert_mongo.insert_one(doc)
def _merge_alert(self, alert_doc):
"""
合并告警到同一个威胁活动
"""
# 查找是否已经生成了威胁活动，没有的话直接退出，然后新增
activity = self.activity.find_record(alert_doc["unique_id"], alert_doc["start_time"])
if not activity:
return False
alert_doc["activity_id"] = activity["_id"]
# 尝试合并告警表单内容完全重复的告警，增加重复次数
if self._merge_repeat_count(alert_doc):
self.activity.update(activity["_id"], {
"$set": {
"end_time": alert_doc["end_time"]
}
})
# 无完全重复，在该威胁活动下新增一条告警
else:
self.activity.add_alert(activity, alert_doc)
self.alert_mongo.insert_one(alert_doc)
return True
def _merge_repeat_count(self, alert_doc: dict) -> bool:
"""
完全重复的告警内容 不再入库 直接统计次数
"""
query = {
"activity_id": alert_doc["activity_id"],
"form_data_id": alert_doc["form_data_id"]
}
record = self.alert_mongo.find_one(query)
if record:
# 账号爆破特殊处理一下
if alert_doc["alert_code"] == "301":
self.alert_mongo.update_one(
{"_id": record["_id"]},
{
"$set": {
"form_data.brute_force_target_users": alert_doc["form_data"]["brute_force_target_users"]
}
}
)
return True
self.alert_mongo.update_one(
{"_id": record["_id"]},
{
"$inc": {"repeat_count": 1},
"$set": {"end_time": alert_doc["end_time"]}
}
)
return True
else:
return False
def _auto_exclude(self, doc):
"""
根据预先设定的规则，自动排除误报
和忽略的区别是，排除的误报不再产生记录，直接忽略
"""
rule_id = self.exclude_rule.match(doc)
if rule_id:
return True
else:
return False
def _auto_ignore(self, doc):
"""
根据预先设定的规则，自动忽略告警
忽略的告警也需要合并
"""
rule_id = self.ignore_rule.match(doc)
if rule_id:
doc["status"] = "auto_ignore"
doc["ignore_rule_id"] = rule_id
return doc
class DetectBase(object):
def __init__(self, code: str, title: str, desc: str):
self.code = code
self.title = title
self.desc = desc
self.log = None
self.krb = None
self.domain = None
self.mongo = MongoHelper(MongoConfig.uri, MongoConfig.db,
MongoConfig.delay_run_collection)
def init(self, log=None, krb=None):
if log is None and krb is None:
raise NoDataInitEvent()
if log and krb:
raise NoDataInitEvent()
if log:
assert isinstance(log, Log)
self.domain = ".".join(log.dc_computer_name.split(".")[1:])
else:
assert isinstance(krb, Kerberos)
self.domain = krb.req.req_body.realm
self.log = log
self.krb = krb
self._format_domain()
def delay_confirm_krb(self, secs=10):
doc = copy.deepcopy(self.krb.record)
doc["_delay_info"] = {
"time": move_n_sec(datetime_now_obj(), -secs),
"data_type": "traffic_kerberos",
"alert_code": self.code
}
self.mongo.insert_one(doc)
def delay_confirm_log(self, secs=60):
doc = copy.deepcopy(self.log.record)
doc["_delay_info"] = {
"time": move_n_sec(datetime_now_obj(), -secs),
"data_type": "event_log",
"alert_code": self.code
}
self.mongo.insert_one(doc)
def _format_domain(self):
if not self.domain:
self.domain = ""
self.netbios_name = ""
self.fqdn_name = ""
elif "." in self.domain:
self.fqdn_name = self.domain
self.netbios_name = _get_netbios_name(self.domain)
elif self.domain and self.domain != "-":
self.netbios_name = self.domain.upper()
self.fqdn_name = _get_fqdn_name(self.domain)
else:
self.netbios_name = ""
self.fqdn_name = ""
def _get_unique_id(self, *args) -> str:
_str = ""
for each in args:
_str += each
return md5(_str)
def _get_log_doc(self):
if self.log is None:
return {}
return self.log.record
def _get_krb_doc(self):
if self.krb is None:
return {}
return self.krb.record
def _get_dc_computer_name(self):
if self.log:
return self.log.dc_computer_name
else:
return self.krb.dc_host_name
def _get_dc_hostname(self):
if self.log:
return self.log.dc_host_name
else:
return self.krb.dc_host_name
def _get_detect_by(self):
if self.log:
return "event_log"
else:
return "krb_traffic"
def _get_time(self):
if self.log:
return utc_to_datetime(self.log.utc_log_time)
else:
return utc_to_datetime(self.krb.utc_time)
def _get_base_doc(self, **kwargs):
return {
"alert_code": self.code,
"title": self.title,
"description": self.desc,
"classify": self._get_classify(),
"dc_computer_name": self._get_dc_computer_name(),
"dc_hostname": self._get_dc_hostname(),
"domain": self.netbios_name,
"status": "pending",
"start_time": self._get_time(),
"end_time": self._get_time(),
"raw_log": self._get_log_doc(),
"raw_krb": self._get_krb_doc(),
"detect_by": self._get_detect_by(),
"repeat_count": 1,
**kwargs
}
def _get_classify(self):
if self.code.startswith("1"):
return "信息探测"
elif self.code.startswith("2"):
return "凭证盗取"
elif self.code.startswith("3"):
return "横向移动"
elif self.code.startswith("4"):
return "权限提升"
elif self.code.startswith("5"):
return "权限维持"
elif self.code.startswith("6"):
return "防御绕过"
else:
return "未知分类"
def _get_workstation_by_source_ip(self, source_ip) -> str:
"""
查找最近该IP认证主机名
返回主机名和域名
"""
account_history = AccountHistory()
return account_history.get_last_workstation_by_ip(source_ip)
def _get_source_ip_by_workstation(self, source_ip) -> str:
"""
"""
account_history = AccountHistory()
return account_history.get_last_ip_by_workstation(source_ip)
def _get_source_ip_by_logon_id(self, logon_id: str, user_name: str) -> str:
id_term = get_term_statement("event_id", 4624)
logon_id_term = get_term_statement("event_data.TargetLogonId.keyword",
logon_id)
user_term = get_term_statement("event_data.TargetUserName.keyword",
user_name)
query = {
"query": get_must_statement(id_term, logon_id_term, user_term),
"_source": ["event_data.IpAddress"],
"size": 1
}
es = ElasticHelper()
rsp = es.search(body=query,
index=ElasticConfig.event_log_index,
doc_type=ElasticConfig.event_log_doc_type)
if rsp and len(rsp["hits"]["hits"]) > 0:
return rsp["hits"]["hits"][0]["_source"]["event_data"]["IpAddress"]
else:
return "unknown"
@abstractmethod
def _generate_alert_doc(self, **kwargs) -> dict:
pass
@abstractmethod
def _get_level(self) -> str:
pass
class Activity(object):
def __init__(self):
self.activity_mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.activities_collection)
self.invasion = Invasion()
def new(self, activity_doc: dict) -> ObjectId:
"""
新增威胁活动记录
同时尝试生成入侵事件记录
"""
invasion_id = self._generate_invasion(activity_doc)
if invasion_id:
activity_doc["invasion_id"] = invasion_id
return self.activity_mongo.insert_one(activity_doc).inserted_id
def add_alert(self, activity_doc, alert_doc: dict):
doc = {
"$set": {}
}
if alert_doc["end_time"] > activity_doc["end_time"]:
doc["$set"]["end_time"] = alert_doc["end_time"]
if alert_doc["level"] > activity_doc["level"]:
doc["$set"]["level"] = alert_doc["level"]
if len(doc["$set"].keys()) == 0:
return
self.activity_mongo.update_one({
"_id": activity_doc["_id"]
}, doc)
def update(self, _id, doc):
self.activity_mongo.update_one({
"_id": _id
}, doc=doc)
def find_record(self, uid, start_time):
"""
根据 unique_id 查找一段时间内相同的威胁活动
"""
return self.activity_mongo.find_one({
"unique_id": uid,
"end_time": {"$gte": start_time + timedelta(hours=-main_config.merge_activity_time)}
})
def _generate_invasion(self, activity_doc) -> ObjectId:
"""
生成入侵事件
多个相同来源IP 不同类型的威胁活动 可以生成一个入侵事件
返回入侵事件的ID
"""
# 首先查找是否已存在对应的入侵事件，尝试合并
invasion_id = self._merge_activity(activity_doc)
if invasion_id:
assert isinstance(invasion_id, ObjectId)
return invasion_id
# 没有已存在的入侵事件，则按照条件，查找是否已存在不同类型的威胁活动
query = {
"form_data.source_ip": activity_doc["form_data"]["source_ip"],
"end_time": {"$gte": activity_doc["start_time"] + timedelta(hours=-main_config.merge_invasion_time)},
"alert_code": {"$ne": activity_doc["alert_code"]}
}
# 如果存在不同类型的威胁活动
another_activities = list(self.activity_mongo.find_all(query).sort("start_time", ASCENDING))
if len(another_activities) > 0:
invasion_id = self.invasion.new(*another_activities, activity_doc)
# 创建完入侵事件之后，将之前查询的到威胁活动全部加上对应的ID
for activity in another_activities:
self.update(activity["_id"], {
"$set": {
"invasion_id": invasion_id
}
})
def _merge_activity(self, activity_doc):
source_ip = activity_doc["form_data"]["source_ip"]
invasion = self.invasion.find_record(source_ip, activity_doc["start_time"])
if not invasion:
return False
# 向存在的入侵事件添加当前威胁活动
self.invasion.add_activity(invasion["_id"], activity_doc)
return invasion["_id"]