class Invasion(object): def __init__(self): self.mongo = MongoHelper(MongoConfig.uri, db=MongoConfig.db, collection=MongoConfig.invasions_collection) def new(self, *activities) -> ObjectId: """ 新建入侵事件 返回入侵事件ID """ source_ip = activities[0]["form_data"]["source_ip"] title = "来自于{ip}的入侵事件".format(ip=source_ip) doc = { "title": title, "level": _get_max_level(list(map(lambda x: x["level"], activities))), "start_time": activities[0]["start_time"], "end_time": activities[-1]["end_time"], "source_ip": source_ip, "status": "pending" } return self.mongo.insert_one(doc).inserted_id def add_activity(self, invasion_id: ObjectId, activity_doc: dict): """ 向当前入侵事件添加一条新的威胁活动 """ query = { "_id": invasion_id } invasion = self.mongo.find_one(query=query) level = _get_max_level([invasion["level"], activity_doc["level"]]) end_time = activity_doc["end_time"] if end_time > invasion["end_time"]: self.update(invasion_id, { "$set": { "level": level, "end_time": end_time } }) else: self.update(invasion_id, { "$set": { "level": level } }) def find_record(self, source_ip, start_time, **kwargs): """ 根据 source_ip 查找一段时间内的相同来源的入侵事件 """ return self.mongo.find_one({ "source_ip": source_ip, "end_time": {"$gte": start_time + timedelta(hours=-main_config.merge_invasion_time)}, **kwargs }) def update(self, _id, doc): self.mongo.update_one({ "_id": _id }, doc=doc)
def init_sensitive_groups(domain): logger.info("init sensitive groups.") domain = get_netbios_domain(domain) ldap_search = LDAPSearch(domain) redis = RedisHelper() mongo = MongoHelper(uri=MongoConfig.uri, db=MongoConfig.db, collection=MongoConfig.settings_collection) sensitive_groups = [] for item in default_sensitive_groups(domain): if len(item["sid"]) > 0: sensitive_groups.append(item) else: entry = ldap_search.search_by_name(item["name"], attributes=["objectSid"]) if not entry or len( entry.entry_attributes_as_dict["objectSid"]) == 0: continue sid = entry.entry_attributes_as_dict["objectSid"][0] item["sid"] = sid sensitive_groups.append(item) logger.info(",".join(list(map(lambda x: x["name"], sensitive_groups)))) sensitive_entry = mongo.find_one({"name": "sensitive_entry"})["value"] sensitive_entry["group"] = sensitive_groups mongo.update_one({"name": "sensitive_entry"}, {"$set": { "value": sensitive_entry }}, upsert=True) redis.set_str_value("sensitive_entry" + REDIS_KEY_SUFFIX, simplejson.dumps(sensitive_entry))
def set_learning_end_time_setting(): value = move_n_days(datetime_utc_now_obj(), 10) logger.info("set learning end time: " + str(value)) name = "learning_end_time" redis = RedisHelper() mongo = MongoHelper(uri=MongoConfig.uri, db=MongoConfig.db, collection=MongoConfig.settings_collection) mongo.update_one(filter={"name": name}, doc={"$set": { "value": value }}, upsert=True) key = name + REDIS_KEY_SUFFIX redis.set_str_value(key, datetime_to_common_str(value))
class Delegation(object): def __init__(self): self.mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.delegation_collection) def new_delegation_record(self, user: User, delegation_type: str, allowed_to=None): if delegation_type not in [ CONSTRAINED_DELEGATION, UNCONSTRAINED_DELEGATION, RES_BASED_CONSTRAINED_DELEGATION ]: raise NoSuchDelegationType() self.mongo.insert_one({ "name": user.user_name, "sid": user.user_sid, "domain": user.domain_name, "delegation_type": delegation_type, "allowed_to": allowed_to }) def find_constrained_delegation_by_sid(self, sid: str): return self.mongo.find_one({ "sid": sid, "delegation_type": CONSTRAINED_DELEGATION }) def find_res_constrained_delegation_by_name(self, name: str): return self.mongo.find_one({ "name": name, "delegation_type": RES_BASED_CONSTRAINED_DELEGATION }) def find_one_delegation(self, query: dict): return self.mongo.find_one(query) def update_delegation(self, sid, delegation_type, allowed_to): self.mongo.update_one({ "sid": sid, "delegation_type": delegation_type }, {"allowed_to": allowed_to})
def init_ldap_settings(domain, server, user, password): logger.info("init the ldap configuration.") if not server.startswith("ldap://"): server = "ldap://" + server mongo = MongoHelper(uri=MongoConfig.uri, db=MongoConfig.db, collection=MongoConfig.settings_collection) query = {"name": "ldap"} doc = { domain: { "server": server, "user": user, "password": password, "dn": get_dn_domain_name(domain) } } mongo.update_one(filter=query, doc={"$set": {"value": doc}}, upsert=True) redis = RedisHelper() redis.set_str_value("ldap" + REDIS_KEY_SUFFIX, simplejson.dumps(doc))
def init_default_settings(domain): logger.info("init other settings.") redis = RedisHelper() mongo = MongoHelper(uri=MongoConfig.uri, db=MongoConfig.db, collection=MongoConfig.settings_collection) for name, value in default_settings.items(): if name == "domain_list": value = [domain] mongo.update_one(filter={"name": name}, doc={"$set": { "value": value }}, upsert=True) key = name + REDIS_KEY_SUFFIX if name in [ "domain_list", "VPN_ip_part", "detail_file_share_white_list_setting" ]: if len(value) != 0: redis.set_list(key, *value) elif name in [ "raw_data_expire", "honeypot_account", "alarms_merge", "sensitive_entry", "kerberos" ]: redis.set_str_value(key, simplejson.dumps(value)) elif name in ["brute_force_max"]: redis.set_str_value(key, str(value)) elif isinstance(value, list): if len(value) > 0 and isinstance(value[0], dict): redis.set_str_value(key, simplejson.dumps(value)) else: if len(value) != 0: redis.set_list(key, *value) elif isinstance(value, str): redis.set_str_value(key, value) elif isinstance(value, dict): redis.set_str_value(key, simplejson.dumps(value)) elif isinstance(value, int): redis.set_str_value(key, str(value))
def get_all_dc_names(domain: str): """ 将DC列表入库 """ domain = get_netbios_domain(domain) logger.info("Search all domain controllers using LDAP.") dc_name_list = [] ldap_search = LDAPSearch(domain) dc_list = ldap_search.search_domain_controller() for each in dc_list: dc_name = str(each["cn"]) dc_name_list.append(dc_name) mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.settings_collection) doc = {domain: dc_name_list} logger.info(",".join(dc_name_list)) logger.info( "domain controller count: {count}".format(count=len(dc_name_list))) logger.info("Save all domain controllers to settings.") mongo.update_one({"name": "dc_name_list"}, {"$set": {"value": doc}}, True) redis = RedisHelper() redis.set_str_value("dc_name_list" + REDIS_KEY_SUFFIX, simplejson.dumps(doc))
class Alert(object): def __init__(self): self.alert_mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.alerts_collection) self.activity = Activity() self.ignore_rule = MatchRules(MongoConfig.ignore_collection) self.exclude_rule = MatchRules(MongoConfig.exclude_collection) def generate(self, doc): """ 生成告警,发送邮件并入库 """ # 计算告警表单内容的唯一ID form_data_id = _get_form_data_md5(doc["form_data"]) doc["form_data_id"] = form_data_id # 误报排除规则过滤,不再产生记录 if self._auto_exclude(doc): return # 忽略规则过滤 doc = self._auto_ignore(doc) # 首先尝试合并相同来源且相同类型的告警到同一个威胁活动, 即unique_id重复的告警 if self._merge_alert(doc): return # 新增,生成威胁活动 activity_id = self.activity.new(doc) # 记录下威胁活动的ID以后,告警入库 doc["activity_id"] = activity_id self.alert_mongo.insert_one(doc) def _merge_alert(self, alert_doc): """ 合并告警到同一个威胁活动 """ # 查找是否已经生成了威胁活动,没有的话直接退出,然后新增 activity = self.activity.find_record(alert_doc["unique_id"], alert_doc["start_time"]) if not activity: return False alert_doc["activity_id"] = activity["_id"] # 尝试合并告警表单内容完全重复的告警,增加重复次数 if self._merge_repeat_count(alert_doc): self.activity.update(activity["_id"], { "$set": { "end_time": alert_doc["end_time"] } }) # 无完全重复,在该威胁活动下新增一条告警 else: self.activity.add_alert(activity, alert_doc) self.alert_mongo.insert_one(alert_doc) return True def _merge_repeat_count(self, alert_doc: dict) -> bool: """ 完全重复的告警内容 不再入库 直接统计次数 """ query = { "activity_id": alert_doc["activity_id"], "form_data_id": alert_doc["form_data_id"] } record = self.alert_mongo.find_one(query) if record: # 账号爆破特殊处理一下 if alert_doc["alert_code"] == "301": self.alert_mongo.update_one( {"_id": record["_id"]}, { "$set": { "form_data.brute_force_target_users": alert_doc["form_data"]["brute_force_target_users"] } } ) return True self.alert_mongo.update_one( {"_id": record["_id"]}, { "$inc": {"repeat_count": 1}, "$set": {"end_time": alert_doc["end_time"]} } ) return True else: return False def _auto_exclude(self, doc): """ 根据预先设定的规则,自动排除误报 和忽略的区别是,排除的误报不再产生记录,直接忽略 """ rule_id = self.exclude_rule.match(doc) if rule_id: return True else: return False def _auto_ignore(self, doc): """ 根据预先设定的规则,自动忽略告警 忽略的告警也需要合并 """ rule_id = self.ignore_rule.match(doc) if rule_id: doc["status"] = "auto_ignore" doc["ignore_rule_id"] = rule_id return doc
class Activity(object): def __init__(self): self.activity_mongo = MongoHelper(MongoConfig.uri, MongoConfig.db, MongoConfig.activities_collection) self.invasion = Invasion() def new(self, activity_doc: dict) -> ObjectId: """ 新增威胁活动记录 同时尝试生成入侵事件记录 """ invasion_id = self._generate_invasion(activity_doc) if invasion_id: activity_doc["invasion_id"] = invasion_id return self.activity_mongo.insert_one(activity_doc).inserted_id def add_alert(self, activity_doc, alert_doc: dict): doc = { "$set": {} } if alert_doc["end_time"] > activity_doc["end_time"]: doc["$set"]["end_time"] = alert_doc["end_time"] if alert_doc["level"] > activity_doc["level"]: doc["$set"]["level"] = alert_doc["level"] if len(doc["$set"].keys()) == 0: return self.activity_mongo.update_one({ "_id": activity_doc["_id"] }, doc) def update(self, _id, doc): self.activity_mongo.update_one({ "_id": _id }, doc=doc) def find_record(self, uid, start_time): """ 根据 unique_id 查找一段时间内相同的威胁活动 """ return self.activity_mongo.find_one({ "unique_id": uid, "end_time": {"$gte": start_time + timedelta(hours=-main_config.merge_activity_time)} }) def _generate_invasion(self, activity_doc) -> ObjectId: """ 生成入侵事件 多个相同来源IP 不同类型的威胁活动 可以生成一个入侵事件 返回入侵事件的ID """ # 首先查找是否已存在对应的入侵事件,尝试合并 invasion_id = self._merge_activity(activity_doc) if invasion_id: assert isinstance(invasion_id, ObjectId) return invasion_id # 没有已存在的入侵事件,则按照条件,查找是否已存在不同类型的威胁活动 query = { "form_data.source_ip": activity_doc["form_data"]["source_ip"], "end_time": {"$gte": activity_doc["start_time"] + timedelta(hours=-main_config.merge_invasion_time)}, "alert_code": {"$ne": activity_doc["alert_code"]} } # 如果存在不同类型的威胁活动 another_activities = list(self.activity_mongo.find_all(query).sort("start_time", ASCENDING)) if len(another_activities) > 0: invasion_id = self.invasion.new(*another_activities, activity_doc) # 创建完入侵事件之后,将之前查询的到威胁活动全部加上对应的ID for activity in another_activities: self.update(activity["_id"], { "$set": { "invasion_id": invasion_id } }) def _merge_activity(self, activity_doc): source_ip = activity_doc["form_data"]["source_ip"] invasion = self.invasion.find_record(source_ip, activity_doc["start_time"]) if not invasion: return False # 向存在的入侵事件添加当前威胁活动 self.invasion.add_activity(invasion["_id"], activity_doc) return invasion["_id"]