def area_info(self, ip, geo_db):
if ipdivide.is_internal_ip(ip):
intranet = "内网".decode('utf-8')
country = "中国".decode('utf-8')
province = ""
city = ""
latitude = ""
longitude = ""
country_code = "cn"
else:
intranet = "外网".decode('utf-8')
country = ""
province = ""
city = ""
latitude = ""
longitude = ""
country_code = ""
try:
reader = geoip2.database.Reader(geo_db)
try:
response = reader.city(ip)
country = response.country.names.get('zh-CN', u'')
province = response.subdivisions.most_specific.names.get(
'zh-CN', u'')
city = response.city.names.get('zh-CN', u'')
latitude = response.location.latitude
longitude = response.location.longitude
country_code = response.country.iso_code
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
if not country:
try:
response = reader.city(ip)
country = response.country.names.get('en', '')
province = response.subdivisions.most_specific.names.get(
'en', '')
city = response.city.names.get('en', '')
latitude = response.location.latitude
longitude = response.location.longitude
country_code = response.country.iso_code
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
ip_geo = {
"intranet": intranet,
"province": province,
"city": city,
"country": country,
"longitude": longitude,
"country_code": str(country_code).lower(),
"latitude": latitude
}
return ip_geo
def capture(self, cfg):
try:
vlan = cfg.get("vlan", "")
pcap = cfg.get("pcap", "")
path = cfg.get("path", "")
access_time = cfg.get("access_time", 20)
passive_cfg = cfg.get("passive_cfg", {})
index_name = passive_cfg.get("es_passive").get("index_name", "")
index_type = passive_cfg.get("es_passive").get("index_type", "")
es_path = passive_cfg.get("es_passive", "").get("path", "")
temp_flow = passive_cfg.get("temp_flow", "temp_flow.csv")
passive_hz = str(passive_cfg.get("ndpi_hz", "5"))
task_type = cfg.get("task_type", "")
IP_addresses = cfg.get("ipaddresses", "")
ip_mode = cfg.get("ip_mode", "")
geo_db = passive_cfg.get("geo_db", "")
keys_list_paths = passive_cfg.get("keys_list_paths", "")
head_key = json.loads(open(keys_list_paths, "r").read())
firms_lib_path = passive_cfg.get("mac_firms", "")
if "vlan" in task_type:
command = path + " -i " + vlan + " -C " + temp_flow + " -m " + passive_hz + " -s " + str(
access_time) + " -q 2>&1 "
else:
command = path + " -i " + pcap + " -C " + temp_flow + " -m " + passive_hz + " -s " + str(
access_time) + " -q 2>&1 "
ip_mode = 3
flow_log.logger.debug("command: " + command)
p = Popen(command, shell=True, stdin=PIPE, stdout=PIPE)
flow_log.logger.debug("ndpi start successful...")
self.work_status = self.Running
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
return 0
'''
path:服务程序启动路径
vlan:被监听的网卡
temp_flow:数据日志
passive_hz:服务程序推送结果频率
access_time:服务启动时间
'''
if self.work_status == self.Running:
obj = esload.ElasticObj(index_name, index_type, ip=es_path)
flow_log.logger.debug('data start combing')
ACTIONS = []
count = 0
fc = threading.Thread(target=self.flowcount.save_data_in_db)
fc.start()
for line in p.stdout:
if not line:
count += 1
time.sleep(1)
if count >= 50:
count = 0
obj.bulk_Index_Data(ACTIONS)
line = line.strip()
if ',' in line:
try:
line = line.encode('utf-8')
temp_datalist = re.split(r",(?![^(]*\))", line)
if len(head_key) > len(temp_datalist):
continue
flow = {}
for index in range(0, len(head_key)):
flow[head_key[index]] = temp_datalist[index]
for key in flow.keys():
if '.' in key:
flow.update(
{key.replace('.', '_'): flow.pop(key)})
try:
category_id = flow.get("protocol_category", "")
category_name = self.category_map[str(
category_id)].decode('utf-8')
except Exception, e:
category_name = "未知".decode('utf-8')
try:
utc_time = datetime.datetime.now().strftime(
"%Y-%m-%dT%H:%M:%S+0800")
flow["timestamp"] = utc_time
flow["device_type"] = category_name
flow["packets"] = int(
flow.get("src2dst_packets", 0)) + int(
flow.get("dst2src_packets", 0))
flow["bytes"] = int(flow.get(
"src2dst_bytes", 0)) + int(
flow.get("dst2src_bytes", 0))
flow["category"] = "软件".decode('utf-8')
flow["vlan"] = vlan
flow["classify"] = ""
flow["last_seen"] = self.timestramp2time(
flow["last_seen"])
flow["first_seen"] = self.timestramp2time(
flow["first_seen"])
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
host_name = str(flow.get("host_name", ""))
if host_name:
if flow.get("detected_app_protocol"
) != "10" and not ipdivide.ipcheck(
host_name):
flow["domain"] = host_name
flow["host_name"] = ""
self.mac_manu_set(flow, flow.get("src_mac", ""), "src",
firms_lib_path)
self.mac_manu_set(flow, flow.get("dest_mac", ""),
"dest", firms_lib_path)
dest_ip_geo = self.set_geo(
str(flow.get("host_b_name")), geo_db)
src_ip_geo = self.set_geo(str(flow.get("host_a_name")),
geo_db)
flow["dest_ip_geo"] = dest_ip_geo
flow["src_ip_geo"] = src_ip_geo
md5str = str(flow.get('host_a_name')) + str(
flow.get(' host_a_port')) + str(
flow.get('host_b_name')) + str(
flow.get('host_b_port'))
_id = self.md5(md5str)
action = {
"_index": obj.index_name,
"_type": obj.index_type,
"_id": _id,
"_source": flow
}
try:
if ipdivide.ipcheck(flow.get(
"host_a_name")) and ipdivide.ipcheck(
flow.get("host_b_name")) and action:
if ip_mode == 2:
if ipdivide.ip_graph_check(
flow.get("host_a_name"),
IP_addresses
) or ipdivide.ip_graph_check(
flow.get("host_b_name"),
IP_addresses):
ACTIONS.append(action)
if len(ACTIONS) > 50:
self.mongo_data(ACTIONS=ACTIONS,
cfg=cfg)
obj.bulk_Index_Data(ACTIONS)
self.outreach(ACTIONS, passive_cfg)
self.flow_counter_model(
self.flowcount, ACTIONS)
ACTIONS = []
elif ip_mode == 1:
if ipdivide.is_internal_ip(
flow.get("host_a_name")
) or ipdivide.is_internal_ip(
flow.get("host_b_name")):
ACTIONS.append(action)
if len(ACTIONS) > 50:
self.mongo_data(ACTIONS=ACTIONS,
cfg=cfg)
obj.bulk_Index_Data(ACTIONS)
self.outreach(ACTIONS, passive_cfg)
self.flow_counter_model(
self.flowcount, ACTIONS)
ACTIONS = []
else:
ACTIONS.append(action)
if len(ACTIONS) > 50:
self.mongo_data(ACTIONS=ACTIONS,
cfg=cfg)
obj.bulk_Index_Data(ACTIONS)
self.outreach(ACTIONS, passive_cfg)
self.flow_counter_model(
self.flowcount, ACTIONS)
ACTIONS = []
else:
# print "invaild ip format "
flow_log.logger.debug(
"invaild ip format," +
str(flow.get("host_a_name")) + ',' +
str(str(flow.get("host_b_name"))))
# print action
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
except:
msg = traceback.format_exc()
flow_log.logger.error(msg)
def redis_data(self, cfg):
try:
ACTIONS = []
flow_log.logger.debug("正在从redis获取数据")
pool = redis.ConnectionPool(host='127.0.0.1', port=6379)
r = redis.Redis(connection_pool=pool)
instrusion_cfg = cfg.get("instrusion_cfg")
es_instrusion = instrusion_cfg.get("es_instrusion")
es_flow = instrusion_cfg.get("es_flow")
obj = esload.ElasticObj(es_instrusion.get("index_name"),
es_instrusion.get("index_type"),
es_instrusion.get("path"))
flow = esload.ElasticObj(es_flow.get("index_name"),
es_flow.get("index_type"),
es_flow.get("path"))
IP_addresses = cfg.get("ipaddresses")
# print cfg
ip_mode = cfg.get("ip_mode")
geo_db = instrusion_cfg.get("geo_db")
classtype_cfg_path = instrusion_cfg.get("classtype_cfg_path", "")
# classtype_cfg_path = "classtype.json"
classtype_dict = json.loads(open(classtype_cfg_path).read())
while True:
try:
if self.task["status"] == 2:
break
try:
ori_data = r.lpop("suricata")
if ori_data:
data1 = json.loads(
json.dumps(ori_data).encode('utf-8'))
data = json.loads(data1)
else:
data = ''
except Exception, e:
data = ''
if not data:
continue
event_type = data.get("event_type")
dest_ip = str(data.get("dest_ip"))
src_ip = str(data.get("src_ip"))
data["dest_ip_geo"] = self.area_info(dest_ip, geo_db)
data["src_ip_geo"] = self.area_info(src_ip, geo_db)
if event_type == 'alert':
classname = data.get("alert", {}).get("category", "")
classtype = classtype_dict.get(classname, {})
data["alert"]["category"] = classtype.get(
"classname", "其他".decode())
data["alert"]["behavior"] = classtype.get(
"behavior", "尝试".decode())
data["alert"]["origin"] = classtype.get(
"origin", "基础防御".decode())
severity = data["alert"]["severity"]
if severity == "1" or severity == 1:
data["alert"]["severity"] = "高危"
elif severity == "2" or severity == 2:
data["alert"]["severity"] = "中危"
else:
data["alert"]["severity"] = "低危"
action = {
"_index": obj.index_name,
"_type": obj.index_type,
"_source": data
}
if ip_mode == 2:
if self.ip_addrs_check(
data.get("dest_ip"),
IP_addresses) or self.ip_addrs_check(
data.get("src_ip"), IP_addresses):
ACTIONS.append(action)
self.mongo_data(ACTIONS=ACTIONS, cfg=cfg)
obj.bulk_Index_Data(ACTIONS)
ACTIONS = []
elif ip_mode == 1:
if ipdivide.is_internal_ip(data.get(
"dest_ip")) or ipdivide.is_internal_ip(
data.get("src_ip")):
ACTIONS.append(action)
self.mongo_data(ACTIONS=ACTIONS, cfg=cfg)
obj.bulk_Index_Data(ACTIONS)
ACTIONS = []
else:
ACTIONS.append(action)
self.mongo_data(ACTIONS=ACTIONS, cfg=cfg)
obj.bulk_Index_Data(ACTIONS)
ACTIONS = []
else:
flow_log.logger.debug("其他类型数据:" + event_type)
# else:
# md5str = str(data.get('dest_ip')) + str(data.get('dest_port')) + str(
# data.get('src_ip')) + str(data.get('src_port'))
# _id = self.md5(md5str)
# action = {
# "_index": flow.index_name,
# "_type": flow.index_type,
# "_source": data
# }
# ACTIONS_flow.append(action)
# if len(ACTIONS_flow)>200:
# flow.bulk_Index_Data(ACTIONS_flow)
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.error(msg)
def mongo_data(self, ACTIONS, cfg):
flow_log.logger.debug("start save in mongo")
passive_cfg = cfg.get("passive_cfg")
mongo_db = passive_cfg.get("mongo_db")
if not mongo_db:
return False
mongo_path = mongo_db.get("path")
mongo_port = mongo_db.get("port")
mongo_index = mongo_db.get("index")
mongo_database = mongo_db.get("database")
mongo_client = mongoclass.Mongoclass(mongo_path, int(mongo_port),
mongo_database)
if not mongo_client.get_state():
flow_log.logger.debug("mongo 数据库连接失败")
return False
if not mongo_client.mongodb_size_count(mongo_index, {}):
flow_log.logger.debug("mongo 数据库数据量限制")
return False
for action in ACTIONS:
try:
flow = action.get("_source")
if flow:
detected_protocol_name = flow.get("detected_protocol_name")
host_a_name = flow.get("host_a_name", "")
host_b_name = flow.get("host_b_name", "")
pc_name = flow.get("host_name", "")
os_name = flow.get("os", "")
mac = flow.get("src_mac", "")
src_manufacturer = flow.get("src_manufacturer", "")
host_list = []
host_list.append(host_a_name)
for host_name in host_list:
if not ipdivide.is_internal_ip(host_a_name):
continue
temp_collect = mongo_client.find(
"passive_flow", {"host_name": host_name})
if temp_collect.count() == 0:
content = {
"tags": [detected_protocol_name],
"host_name": host_name,
"pc_name": pc_name,
"os_name": os_name,
"mac": mac,
"src_manufacturer": src_manufacturer
}
mongo_client.insert_one(mongo_index, content)
else:
tags = [detected_protocol_name]
for temp in temp_collect:
tag = temp.get("tags")
if isinstance(tag, list):
tags.extend(tag)
tags = list(set(tags))
content = {"host_name": host_name, "tags": tags}
if pc_name:
content["pc_name"] = pc_name
else:
content["pc_name"] = ""
if os_name:
content["os_name"] = os_name
else:
content["os_name"] = ""
if temp_collect.count() != 1:
mongo_client.delete(mongo_index,
{"host_name": host_name})
mongo_client.insert_one(mongo_index, content)
else:
mongo_client.update_one(
mongo_index, [{
"host_name": host_name
}, content])
else:
return False
except Exception, e:
msg = traceback.format_exc()
flow_log.logger.error(msg)