def read_flow_from_db(self):
try:
mongo_db = self.passive_cfg.get("mongo_db")
if not mongo_db:
return False, 0
mongo_path = mongo_db.get("path")
mongo_port = mongo_db.get("port")
mongo_index = str(mongo_db.get("index")) + "flowcounter"
mongo_database = mongo_db.get("database")
mongo_client = mongoclass.Mongoclass(mongo_path, int(mongo_port),
mongo_database)
if not mongo_client.get_state():
flow_log.logger.debug("mongo 数据库连接失败")
return False
if not mongo_client.mongodb_size_count(mongo_index, {}):
flow_log.logger.debug("mongo 数据库数据量限制")
return False
all_data = mongo_client.find(mongo_index, {})
for data in all_data:
try:
data.pop('_id')
self.flow_all.append(data)
except:
msg = traceback.format_exc()
print msg
return True, len(self.flow_all)
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.debug(msg)
return True, 0
def blackip_path_check(self):
try:
if not self.path or self.port == 0 or not self.index or not self.database:
return False
mongo_client = mongoclass.Mongoclass(self.path, self.port,
self.database)
if not mongo_client.get_state():
return False
return True
except Exception, e:
print e
return False
def blackip_get(self):
try:
mongo_client = mongoclass.Mongoclass(self.path, self.port,
self.database)
if not self.blackip_path_check():
return False
total_data = mongo_client.find(self.index + "blackip", {})
blackips = []
for data in total_data:
blackip = data.get("ipaddr", "")
if blackip and self.ipcheck(blackip):
blackips.append(blackip)
return blackips
except Exception, e:
return []
def blackdomain_get(self):
try:
mongo_client = mongoclass.Mongoclass(self.path, self.port,
self.database)
if not self.blackdomain_path_check():
return False
total_data = mongo_client.find(self.index + "blackdomain", {})
domains = []
for data in total_data:
rew = {}
content = data.get("domain", "")
classtype = data.get("reason", "")
block_type = int(data.get("type"))
create_time = data.get("create_time")
if content and classtype and block_type in range(-1, 4):
rew["content"] = content
rew["classtype"] = classtype
rew["type"] = block_type
rew["create_time"] = create_time
domains.append(rew)
return domains
except Exception, e:
return []
def save_data_in_db(self):
while self.status:
try:
time.sleep(self.sleep_time)
mongo_db = self.passive_cfg.get("mongo_db", "")
if not mongo_db:
flow_log.logger.debug("mongo 数据库连接失败")
continue
mongo_path = mongo_db.get("path")
mongo_port = mongo_db.get("port")
mongo_index = str(mongo_db.get("index")) + "flowcounter"
mongo_database = mongo_db.get("database")
mongo_client = mongoclass.Mongoclass(mongo_path,
int(mongo_port),
mongo_database)
if not mongo_client.get_state():
flow_log.logger.debug("mongo 数据库连接失败")
continue
if not mongo_client.mongodb_size_count(mongo_index, {}):
flow_log.logger.debug("mongo 数据库数据量限制")
continue
for content in self.wait_update:
try:
tag = content.get("tag", "")
if tag not in [
"all", "ipaddr", "protocol", "event", "vlan",
"last_seen"
]:
continue
if tag == "all":
condition = {"tag": "all"}
mongo_client.update_one(mongo_index,
[condition, content])
else:
object_name = content.get("object", "")
condition = {"tag": tag, "object": object_name}
mongo_client.update_one(mongo_index,
[condition, content])
except:
msg = traceback.format_exc()
flow_log.logger.debug(msg)
try:
for content in self.wait_insert:
tag = content.get("tag", "")
if tag not in [
"all", "ipaddr", "protocol", "event", "vlan",
"last_seen"
]:
self.wait_insert.remove(content)
continue
if len(self.wait_insert) > 0:
mongo_client.insert_many(mongo_index, self.wait_insert)
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.debug(msg)
self.wait_update = []
self.wait_insert = []
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.debug(msg)
def mongo_data(self, ACTIONS, cfg):
'''
:param action:
:param cfg:
:return:
'''
for action in ACTIONS:
try:
instrusion_cfg = cfg.get("instrusion_cfg")
mongo_db = instrusion_cfg.get("mongo_db")
if not mongo_db:
return False
mongo_path = mongo_db.get("path")
mongo_port = mongo_db.get("port")
mongo_database = mongo_db.get("database")
mongo_index = mongo_db.get("index")
mongo_index_sign = str(mongo_db.get("index")) + "sign"
mongo_index_event = str(mongo_db.get("index")) + "event"
mongo_client = mongoclass.Mongoclass(mongo_path,
int(mongo_port),
mongo_database)
if not mongo_client.get_state():
flow_log.logger.debug("mongo 数据库连接失败")
return False
if not mongo_client.mongodb_size_count(mongo_index, {}):
flow_log.logger.debug("mongo 数据库数据量限制")
return False
event = action.get("_source")
if event:
alert = event.get("alert")
dest_ip = event.get("dest_ip")
src_ip = event.get("src_ip")
timestamp = event.get("timestamp")
# print timestamp
try:
if alert:
signature = alert.get("signature")
if signature:
temp = mongo_client.find(mongo_index_sign,
{"signature": signature})
if temp.count() == 0:
content = {
"host_names": [src_ip, dest_ip],
"signature": signature,
}
mongo_client.insert_one(
mongo_index_sign, content)
else:
host_names = [src_ip, dest_ip]
for row in temp:
host = row.get("host_names")
if isinstance(host, list):
host_names.extend(host)
host_names = list(set(host_names))
content = {
"host_names": host_names,
"signature": signature
}
if temp.count() > 1:
mongo_client.delete(
mongo_index_sign,
{"signature": signature})
mongo_client.insert_one(
mongo_index_sign, content)
else:
mongo_client.update_one(
mongo_index_sign,
[{
"signature": signature
}, content])
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.debug(msg)
try:
host_list = []
host_list.append(src_ip)
host_list.append(dest_ip)
for host_name in host_list:
temp_collect = mongo_client.find(
mongo_index, {"host_name": host_name})
if temp_collect.count() == 0:
content = {
"timestamp": timestamp,
"times": 1,
"host_name": host_name,
}
mongo_client.insert_one(mongo_index, content)
else:
times = 1
for temp in temp_collect:
temp_times = int(temp.get("times"))
times += temp_times
content = {
"timestamp": timestamp,
"times": times,
"host_name": host_name,
}
if temp_collect.count() > 1:
mongo_client.delete(
mongo_index, {"host_name": host_name})
mongo_client.insert_one(
mongo_index, content)
else:
mongo_client.update_one(
mongo_index, [{
"host_name": host_name
}, content])
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.debug(msg)
try:
temp_collect = mongo_client.find(
mongo_index_event, {"signature": signature})
if temp_collect.count() == 0:
content = {
"timestamp": timestamp,
"times": 1,
"signature": signature,
}
mongo_client.insert_one(mongo_index_event, content)
else:
times = 1
for temp in temp_collect:
temp_times = int(temp.get("times"))
times += temp_times
content = {
"timestamp": timestamp,
"times": times,
"signature": signature,
}
if temp_collect.count() > 1:
mongo_client.delete(mongo_index_event,
{"signature": signature})
mongo_client.insert_one(
mongo_index_event, content)
else:
mongo_client.update_one(
mongo_index_event, [{
"signature": signature
}, content])
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.debug(msg)
def outreach(self, ACTIONS, passive_cfg):
try:
mongo_db = passive_cfg.get("mongo_db")
if not mongo_db:
return False
mongo_path = mongo_db.get("path")
mongo_port = mongo_db.get("port")
mongo_index = str(mongo_db.get("index")) + "outreach"
mongo_database = mongo_db.get("database")
mongo_client = mongoclass.Mongoclass(mongo_path, int(mongo_port),
mongo_database)
if not mongo_client.get_state():
flow_log.logger.debug("mongo 数据库连接失败")
return False
if not mongo_client.mongodb_size_count(mongo_index, {}):
flow_log.logger.debug("mongo 数据库数据量限制")
return False
for action in ACTIONS:
flow = action.get("_source", {})
content = {}
domain = flow.get("domain", "")
detected_app_protocol = str(
flow.get("detected_app_protocol", ""))
if domain and detected_app_protocol != '5':
similardoamin, domain_classtype = self.search_domain_classtype(
domain)
if domain_classtype:
content["classtype"] = domain_classtype
else:
content["classtype"] = ""
first_seen = flow.get("first_seen", "")
last_seen = flow.get("last_seen", "")
dest_ip_geo = flow.get("dest_ip_geo", {})
dst2src_bytes = float(flow.get("dst2src_bytes", ""))
src2dst_bytes = float(flow.get("src2dst_bytes", ""))
detected_protocol_name = flow.get("detected_protocol_name",
"")
country_code = dest_ip_geo.get("country_code", "")
if country_code != "cn":
domain_geo = 1
else:
domain_geo = 0
host_b_name = flow.get("host_b_name", "")
host_a_name = flow.get("host_a_name", "")
content["domain_geo"] = domain_geo
content["first_seen"] = first_seen
content["last_seen"] = last_seen
content["country_code"] = dest_ip_geo.get(
"country_code", "")
content["country"] = dest_ip_geo.get("country", "")
content["dst2src_bytes"] = dst2src_bytes
content["src2dst_bytes"] = src2dst_bytes
content["protocol_name"] = detected_protocol_name
content["domain"] = domain
content["source_ip"] = host_a_name
temp_collect = mongo_client.find(mongo_index, {
"domain": domain,
"source_ip": host_a_name
})
if temp_collect.count() == 0:
content["dest_ip"] = [host_b_name]
content["times"] = 1
content["orid"] = str(uuid.uuid4()).decode("utf-8")
mongo_client.insert_one(mongo_index, content)
else:
source_ip_list = []
for data in temp_collect:
dst2src_bytes += float(data.get("dst2src_bytes"))
src2dst_bytes += float(data.get("src2dst_bytes"))
source_ip_list = data.get("content", [])
source_ip_list.append(host_b_name)
times = int(data.get("times"))
times += 1
content["orid"] = data.get("orid")
content["times"] = times
content["dest_ip"] = source_ip_list
content["dst2src_bytes"] = dst2src_bytes
content["src2dst_bytes"] = src2dst_bytes
mongo_client.update_one(mongo_index,
[{
"domain": domain,
"source_ip": host_a_name
}, content])
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
def mongo_data(self, ACTIONS, cfg):
flow_log.logger.debug("start save in mongo")
passive_cfg = cfg.get("passive_cfg")
mongo_db = passive_cfg.get("mongo_db")
if not mongo_db:
return False
mongo_path = mongo_db.get("path")
mongo_port = mongo_db.get("port")
mongo_index = mongo_db.get("index")
mongo_database = mongo_db.get("database")
mongo_client = mongoclass.Mongoclass(mongo_path, int(mongo_port),
mongo_database)
if not mongo_client.get_state():
flow_log.logger.debug("mongo 数据库连接失败")
return False
if not mongo_client.mongodb_size_count(mongo_index, {}):
flow_log.logger.debug("mongo 数据库数据量限制")
return False
for action in ACTIONS:
try:
flow = action.get("_source")
if flow:
detected_protocol_name = flow.get("detected_protocol_name")
host_a_name = flow.get("host_a_name", "")
host_b_name = flow.get("host_b_name", "")
pc_name = flow.get("host_name", "")
os_name = flow.get("os", "")
mac = flow.get("src_mac", "")
src_manufacturer = flow.get("src_manufacturer", "")
host_list = []
host_list.append(host_a_name)
for host_name in host_list:
if not ipdivide.is_internal_ip(host_a_name):
continue
temp_collect = mongo_client.find(
"passive_flow", {"host_name": host_name})
if temp_collect.count() == 0:
content = {
"tags": [detected_protocol_name],
"host_name": host_name,
"pc_name": pc_name,
"os_name": os_name,
"mac": mac,
"src_manufacturer": src_manufacturer
}
mongo_client.insert_one(mongo_index, content)
else:
tags = [detected_protocol_name]
for temp in temp_collect:
tag = temp.get("tags")
if isinstance(tag, list):
tags.extend(tag)
tags = list(set(tags))
content = {"host_name": host_name, "tags": tags}
if pc_name:
content["pc_name"] = pc_name
else:
content["pc_name"] = ""
if os_name:
content["os_name"] = os_name
else:
content["os_name"] = ""
if temp_collect.count() != 1:
mongo_client.delete(mongo_index,
{"host_name": host_name})
mongo_client.insert_one(mongo_index, content)
else:
mongo_client.update_one(
mongo_index, [{
"host_name": host_name
}, content])
else:
return False
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.error(msg)