def getWorldChart():
es = Elasticsearch([{
'host': app.config['ELASTICSEARCH_URI'],
'port': int(app.config['ELASTICSEARCH_PORT'])
}])
timeSetting = request.args['timeSetting']
edTime = parser.parse(timeSetting) + datetime.timedelta(days=1)
str_dt = Local2UTC(parser.parse(timeSetting)).isoformat()
end_dt = Local2UTC(edTime).isoformat()
body = getWorldChartQuery(
str_dt, end_dt, app.config['ANALYSIS_RESULTS_SECURITY_LEVEL_MIN'])
res = es.search(index=app.config['ELASTICSEARCH_INDEX'],
doc_type="analysis_results",
body=body,
request_timeout=30)
mapData = []
latlong = dict()
i = 0
for doc in res['aggregations']['group_by_country2']['buckets']:
if doc['key'] == '':
continue
_nation = (_nation for _nation in nations
if _nation["code"] == doc['key']).next()
mapData.append({
"code": doc['key'],
"name": _nation['nation'],
'value': doc['doc_count'],
'color': colorlist[i]
})
if i >= colorlist.__len__() - 1:
i = 0
else:
i = i + 1
latlong[doc['key']] = {
"latitude": _nation['latitude'],
"longitude": _nation['longitude']
}
# mapData = []
# latlong = dict()
# mapData.append({"code": 'KR', "name": "korea", 'value': 6, 'color': colorlist[0]})
# mapData.append({"code" : 'CN', "name" : "china", 'value' : 21, 'color' : colorlist[1] } )
# mapData.append({"code": 'US', "name": "us", 'value': 7, 'color': colorlist[2]})
# latlong['KR'] = { "latitude" : 37.00, "longitude" : 127.30 }
# latlong['CN'] = {"latitude": 35.00, "longitude": 105.00}
# latlong['US'] = {"latitude": 38.00, "longitude": -97.00}
chartdata = OrderedDict()
chartdata['latlong'] = latlong
chartdata['mapData'] = mapData
return json.dumps(chartdata)
def GetLinkDnaListQueryEs(request, per_page = None, start_idx = None, add_ip = None):
str_dt = ""
end_dt = ""
if request.form.get('timeFrom') is not None and request.form.get('timeFrom') != "":
str_dt = Local2UTC(parser.parse(request.form['timeFrom'])).isoformat()
else:
str_dt = (datetime.utcnow()- timedelta(hours=1)).isoformat()
if request.form.get('timeTo') is not None and request.form['timeTo'] != "":
end_dt = Local2UTC(parser.parse(request.form['timeTo'])).isoformat()
else:
end_dt = datetime.utcnow().isoformat()
if request.form.get('timeSpan') is not None and request.form.get('timeSpan') != "":
str_dt = "now-"+request.form.get('timeSpan')
end_dt = "now"
if per_page == None:
per_page = int(request.form['perpage'])
if start_idx == None:
start_idx = int(request.form['start'])
page_no = int(start_idx / per_page)
#region get parameter
search_data_type = request.form.get("search_data_type")
search_src_ip = request.form.get('search_src_ip')
search_src_ip_opt = request.form.get('search_src_ip_opt')
search_dst_ip = request.form.get('search_dst_ip')
search_dst_ip_opt = request.form.get('search_dst_ip_opt')
search_src_port_value = request.form.get('search_src_port_value')
search_src_port_opt = request.form.get('search_src_port_opt')
search_dst_port_value = request.form.get('search_dst_port_value')
search_dst_port_opt = request.form.get('search_dst_port_opt')
search_protocol_opt = request.form.get('search_protocol_opt')
search_protocol_value = request.form.get('search_protocol_value')
search_packet_opt = request.form['search_packet_opt']
search_packet_value = request.form['search_packet_value']
search_bytes_opt = request.form['search_bytes_opt']
search_bytes_value = request.form['search_bytes_value']
flag_urg_opt = request.form['search_flag_urg_opt']
flag_urg_value = request.form['search_flag_urg_value']
flag_ack_opt = request.form['search_flag_ack_opt']
flag_ack_value = request.form['search_flag_ack_value']
flag_psh_opt = request.form['search_flag_psh_opt']
flag_psh_value = request.form['search_flag_psh_value']
flag_rst_opt = request.form['search_flag_rst_opt']
flag_rst_value = request.form['search_flag_rst_value']
flag_syn_opt = request.form['search_flag_syn_opt']
flag_syn_value = request.form['search_flag_syn_value']
flag_fin_opt = request.form['search_flag_fin_opt']
flag_fin_value = request.form['search_flag_fin_value']
geo_distance_opt = request.form['search_geo_distance_opt']
geo_distance_value = request.form['search_geo_distance_value']
search_src_country = request.form.get('search_src_country')
search_dst_country = request.form.get('search_dst_country')
search_syslog_name_opt = request.form.get('search_syslog_name_opt')
search_syslog_name_value = request.form.get('search_syslog_name_value')
search_msg_opt = request.form.get('search_msg_opt')
search_msg_value = request.form.get('search_msg_value')
#endregion
query = """
select * FROM {0}
where
1 = 1
"""
mustlist = []
notmustlist = []
shouldlist = []
#region make query
if str_dt != "" and end_dt != "":
query = { "range":{"min_timestamp": {"gte": str_dt, "lte": end_dt}} }
mustlist.append(query)
if search_data_type != "" and search_data_type is not None:
query = CreateFilterQuery_IP("=", "data_type", search_data_type)
mustlist.append(query)
if search_src_ip != "" and search_src_ip is not None and search_src_ip_opt != "":
query = CreateFilterQuery_IP(search_src_ip_opt, "src_ip", search_src_ip)
if search_src_ip_opt != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_dst_ip != ""and search_dst_ip is not None and search_dst_ip_opt != "":
query = CreateFilterQuery_IP(search_dst_ip_opt, "dst_ip", search_dst_ip)
if search_dst_ip_opt != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_src_port_value is not None and search_src_port_value != "" :
query = CreateFilterQuery(search_src_port_opt, "src_port", search_src_port_value)
if search_src_port_opt != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_dst_port_value is not None and search_dst_port_value != "" :
query = CreateFilterQuery(search_dst_port_opt, "dst_port", search_dst_port_value)
if search_dst_port_opt != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_protocol_value is not None and search_protocol_value != "" :
query = CreateFilterQuery(search_protocol_opt, "protocol", search_protocol_value)
if search_protocol_opt != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_packet_opt != "" and search_packet_value != "" :
query = CreateFilterQuery(search_packet_opt, "in_pkts", search_packet_value)
if search_packet_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_bytes_opt != "" and search_bytes_value != "":
query = CreateFilterQuery(search_bytes_opt, "in_bytes", search_bytes_value)
if search_bytes_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if flag_urg_opt != "" and flag_urg_value != "":
query = CreateFilterQuery(flag_urg_opt, "tcp_flags_URG", flag_urg_value)
if flag_urg_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if flag_ack_opt != "" and flag_ack_value != "":
query = CreateFilterQuery(flag_ack_opt, "tcp_flags_ACK", flag_ack_value)
if flag_ack_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if flag_psh_opt != "" and flag_psh_value != "":
query = CreateFilterQuery(flag_psh_opt, "tcp_flags_PSH", flag_psh_value)
if flag_psh_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if flag_rst_opt != "" and flag_rst_value != "":
query = CreateFilterQuery(flag_rst_opt, "tcp_flags_RST", flag_rst_value)
if flag_rst_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if flag_syn_opt != "" and flag_syn_value != "":
query = CreateFilterQuery(flag_syn_opt, "tcp_flags_SYN", flag_syn_value)
if flag_syn_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if flag_fin_opt != "" and flag_fin_value != "":
query = CreateFilterQuery(flag_fin_opt, "tcp_flags_FIN", flag_fin_value)
if flag_fin_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if geo_distance_opt != "" and geo_distance_value != "":
query = CreateFilterQuery(geo_distance_opt, "geoip_distance", geo_distance_value)
if geo_distance_value != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_src_country is not None and search_src_country != "" :
query = CreateFilterQuery("=", "src_country_code.keyword", search_src_country)
mustlist.append(query)
if search_dst_country is not None and search_dst_country != "" :
query = CreateFilterQuery("=", "dst_country_code.keyword", search_dst_country)
mustlist.append(query)
if search_syslog_name_value is not None and search_syslog_name_value != "" :
query = CreateFilterQuery(search_syslog_name_opt, "syslog_name.keyword", search_syslog_name_value)
if search_syslog_name_opt != "!=":
mustlist.append(query)
else:
notmustlist.append(query)
if search_msg_value is not None and search_msg_value != "" :
query = CreateFilterQuery(search_msg_opt, "msg", "*" + search_msg_value + "*")
if search_msg_opt == "like":
mustlist.append(query)
else:
notmustlist.append(query)
#endregion
completeQuery = CreateEsQuery(per_page, page_no*per_page, mustlist, notmustlist, shouldlist )
return completeQuery
def netflow_es(datasource):
recursiveSearchLvl = 3
search_ip = request.args['search_ip']
max_nodes_size = int(request.args['max_nodes_size'])
tempTable = 'tab1'
es = Elasticsearch([{
'host': app.config['ELASTICSEARCH_URI'],
'port': app.config['ELASTICSEARCH_PORT']
}])
body = GetLinkDnaListQueryEs(request, max_nodes_size, 0)
res = None
if request.form.get('search_src_ip') == "" and request.form.get(
'search_dst_ip') == "":
res = es.search(index="gsp-*",
doc_type="link_dna_tuple4",
body=body,
request_timeout=30)
else:
for i in range(0, recursiveSearchLvl):
if res is not None and res['hits']['hits'].__len__(
) >= max_nodes_size:
break
res = NetLinkReSearch(es, res, max_nodes_size)
chartdata = OrderedDict()
nodes = list()
links = list()
uniquenodelist = []
max_nodevalue = 1
max_link = dict()
max_link.update({
'bytes': 1,
'pkts': 1,
'session_cnt': 1,
'tcp_flags_URG': 1,
'tcp_flags_ACK': 1,
'tcp_flags_PSH': 1,
'tcp_flags_RST': 1,
'tcp_flags_SYN': 1,
'tcp_flags_FIN': 1,
'tcp_flags_ACK': 1
})
for doc in res['hits']['hits']:
row = doc['_source']
if row['src_ip'] == None \
or row['dst_ip'] == None:
continue
if row['data_type'] == 'netflows':
#Link Value 최대값 저장 (선 굵기 비율 계산용)
if max_link['bytes'] < float(row['bytes']):
max_link['bytes'] = float(row['bytes'])
if max_link['pkts'] < float(row['pkts']):
max_link['pkts'] = float(row['pkts'])
if max_link['tcp_flags_URG'] < float(row['tcp_flags_URG']):
max_link['tcpflags_URG'] = float(row['tcpflags_URG'])
if max_link['tcp_flags_ACK'] < float(row['tcp_flags_ACK']):
max_link['tcp_flags_ACK'] = float(row['tcp_flags_ACK'])
if max_link['tcp_flags_PSH'] < float(row['tcp_flags_PSH']):
max_link['tcp_flags_PSH'] = float(row['tcp_flags_PSH'])
if max_link['tcp_flags_RST'] < float(row['tcp_flags_RST']):
max_link['tcp_flags_RST'] = float(row['tcp_flags_RST'])
if max_link['tcp_flags_SYN'] < float(row['tcp_flags_SYN']):
max_link['tcp_flags_SYN'] = float(row['tcp_flags_SYN'])
if max_link['tcp_flags_FIN'] < float(row['tcp_flags_FIN']):
max_link['tcp_flags_FIN'] = float(row['tcp_flags_FIN'])
srcNodeName = str(row['src_ip'])
isExists = next((index for (index, d) in enumerate(uniquenodelist)
if d["name"] == srcNodeName), None)
if isExists is None:
node_src = dict()
node_src['name'] = srcNodeName
node_src['ip'] = row['src_ip']
node_src['value'] = getValueAlter(row, 'session_cnt', 1) #원 크기
node_src['color'] = "#FF7F27"
node_src['type'] = 'Source'
uniquenodelist.append(node_src)
if max_nodevalue < int(getValueAlter(node_src, 'value', 1)):
max_nodevalue = int(getValueAlter(node_src, 'value', 1))
else:
uniquenodelist[isExists]['value'] += getValueAlter(
row, 'session_cnt', 1) #원 크기
if max_nodevalue < float(uniquenodelist[isExists]['value']):
max_nodevalue = float(uniquenodelist[isExists]['value'])
dstNodeName = row['dst_ip']
isExists = next((index for (index, d) in enumerate(uniquenodelist)
if d["name"] == dstNodeName), None)
if isExists is None:
node_dst = dict()
node_dst['name'] = dstNodeName
node_dst['ip'] = row['dst_ip']
node_dst['value'] = getValueAlter(row, 'session_cnt', 1) #원 크기
node_dst['color'] = "blue"
node_dst['type'] = 'destination'
uniquenodelist.append(node_dst)
if max_nodevalue < int(getValueAlter(node_dst, 'value', 1)):
max_nodevalue = int(getValueAlter(node_dst, 'value', 1))
else:
uniquenodelist[isExists]['value'] += getValueAlter(
row, 'session_cnt', 1) # 원 크기
if max_nodevalue < float(uniquenodelist[isExists]['value']):
max_nodevalue = float(uniquenodelist[isExists]['value'])
#uniquenodelist = list(set(uniquenodelist))
for doc in res['hits']['hits']:
row = doc['_source']
if row['dst_ip'] == None \
or row['src_ip'] == None:
continue
timestamp = Local2UTC(parser.parse(
row['session_min_time'])).strftime("%Y-%m-%d %H:%M:%S")
targetIdx = next((index for (index, d) in enumerate(uniquenodelist)
if d["name"] == str(row['dst_ip'])), None)
sourceIdx = next((index for (index, d) in enumerate(uniquenodelist)
if d["name"] == str(row['src_ip'])), None)
distance = getValueAlter(row, 'geoip_distance', None)
link = {
"source": targetIdx, # src_ip
"target": sourceIdx, # dst_ip
"timestamp": timestamp,
"value": getLineWidth(row.get('bytes'), max_link.get('bytes')),
"s_count": row.get('session_cnt'),
"s_time": row.get('differ_time'),
"bytes": row.get('bytes'),
"pkts": row.get('pkts'),
"tcp_urg": row.get('tcp_flags_URG'),
"tcp_ack": row.get('tcp_flags_ACK'),
"tcp_psh": row.get('tcp_flags_PSH'),
"tcp_rst": row.get('tcp_flags_RST'),
"tcp_syn": row.get('tcp_flags_SYN'),
"tcp_fin": row.get('tcp_flags_FIN'),
"tcp_urg_p": row.get('tcp_flags_URG_P'),
"tcp_ack_p": row.get('tcp_flags_ACK_P'),
"tcp_psh_p": row.get('tcp_flags_PSH_P'),
"tcp_rst_p": row.get('tcp_flags_RST_P'),
"tcp_syn_p": row.get('tcp_flags_SYN_P'),
"tcp_fin_p": row.get('tcp_flags_FIN_P'),
"distance": distance,
"protocol": row.get('protocol'),
"profile": row.get('profile')
#,"svrip_base_conn_cnt": row['svrip_base_conn_cnt'],
#"clip_base_conn_cnt": row['clip_base_conn_cnt']
}
links.append(link)
for _node in uniquenodelist:
node = {
"name": _node['name'].replace(' ', ''),
"ip": _node['ip'],
"value": getCircleSize(float(_node['value']), max_nodevalue),
"color": _node['color'],
"conCount": _node['value'],
"type": _node['type']
}
nodes.append(node)
chartdata["nodes"] = nodes
chartdata["links"] = links
chartdata['maxdata'] = max_link
return json.dumps(chartdata)
def getLinkDnaDetailAnalaysis():
searchtime = Local2UTC(parser.parse(request.args['search_time']))
endtime = searchtime + datetime.timedelta(hours=12)
starttime = searchtime - datetime.timedelta(hours=12)
svr_ip = request.args['svr_ip']
svr_port = request.args['svr_port']
cl_ip = request.args['cl_ip']
query = {
"size": 10,
"sort": [{
"security_level": "desc"
}],
"query": {
"bool": {
"must": [{
"range": {
"@timestamp": {
"gte": starttime.isoformat(),
"lte": endtime.isoformat()
}
}
}, {
"term": {
"svr_ip": svr_ip
}
}, {
"term": {
"cl_ip": cl_ip
}
}, {
"term": {
"svr_port": svr_port
}
}]
}
}
}
result = dict()
es = Elasticsearch([{
'host': app.config['ELASTICSEARCH_URI'],
'port': int(app.config['ELASTICSEARCH_PORT'])
}])
res = es.search(index=app.config['ELASTICSEARCH_INDEX'],
doc_type="analysis_results",
body=query,
request_timeout=30)
result['file_name'] = ''
result['uri'] = ''
result['md5'] = ''
result['security_level'] = ''
for _item in res['hits']['hits']:
result['file_name'] = _item['_source']['file_name']
result['uri'] = _item['_source']['uri']
result['md5'] = _item['_source']['md5']
if int(_item['_source']['security_level']) >= int(
app.config['ANALYSIS_RESULTS_SECURITY_LEVEL_MIN']):
result['security_level'] = "악성"
else:
result['security_level'] = "정상"
break
str_json = json.dumps(result)
return Response(str_json, mimetype='application/json')
def GetNetLinkChartQuery(request, tblName):
str_dt = ""
end_dt = ""
if request.form['timeFrom'] != "":
utcTime = Local2UTC(parser.parse(request.form['timeFrom']))
str_dt = utcTime.strftime('%Y-%m-%d %H:%M:%S')
if request.form['timeTo'] != "":
utcTime = Local2UTC(parser.parse(request.form['timeTo']))
end_dt = utcTime.strftime('%Y-%m-%d %H:%M:%S')
search_ip = request.form['search_ip']
search_ip_opt = request.form['search_ip_opt']
sendbyte_opt = request.form['search_sendbyte_opt']
sendbyte_value = request.form['search_sendbyte_value']
receivebyte_opt = request.form['search_receivebyte_opt']
receivebyte_value = request.form['search_receivebyte_value']
sendpacket_opt = request.form['search_sendpacket_opt']
sendpacket_value = request.form['search_sendpacket_value']
receive_packet_opt = request.form['search_receive_packet_opt']
receive_packet_value = request.form['search_receive_packet_value']
session_time_opt = request.form['search_session_time_opt']
session_time_value = request.form['search_session_time_value']
session_count_opt = request.form['search_session_count_opt']
session_count_value = request.form['search_session_count_value']
flag_urg_opt = request.form['search_flag_urg_opt']
flag_urg_value = request.form['search_flag_urg_value']
flag_ack_opt = request.form['search_flag_ack_opt']
flag_ack_value = request.form['search_flag_ack_value']
flag_psh_opt = request.form['search_flag_psh_opt']
flag_psh_value = request.form['search_flag_psh_value']
flag_rst_opt = request.form['search_flag_rst_opt']
flag_rst_value = request.form['search_flag_rst_value']
flag_syn_opt = request.form['search_flag_syn_opt']
flag_syn_value = request.form['search_flag_syn_value']
flag_fin_opt = request.form['search_flag_fin_opt']
flag_fin_value = request.form['search_flag_fin_value']
svr_con_count_opt = request.form['search_svr_con_count_opt']
svr_con_count_value = request.form['search_svr_con_count_value']
cli_con_count_opt = request.form['search_cli_con_count_opt']
cli_con_count_value = request.form['search_cli_con_count_value']
geo_distance_opt = request.form['search_geo_distance_opt']
geo_distance_value = request.form['search_geo_distance_value']
query = """
select * FROM {0}
where
1 = 1
"""
query = query.format(tblName)
if str_dt != "" and end_dt != "":
query += "and end_time between '{0}' and '{1}' \n".format(
str_dt, end_dt)
if search_ip != "" and search_ip_opt == "=":
query += "and ( cl_ip = '{0}' or svr_ip = '{0}' )\n".format(search_ip)
elif search_ip != "" and search_ip_opt == "!=":
query += "and ( cl_ip <> '{0}' and svr_ip <> '{0}' )\n".format(
search_ip)
if sendbyte_opt != "" and sendbyte_value != "":
query += "and send_bytes {0} {1} \n".format(sendbyte_opt,
sendbyte_value)
if receivebyte_opt != "" and receivebyte_value != "":
query += "and recv_bytes {0} {1} \n".format(receivebyte_opt,
receivebyte_value)
if sendpacket_opt != "" and sendpacket_value != "":
query += "and send_pkts {0} {1} \n".format(sendpacket_opt,
sendpacket_value)
if receive_packet_opt != "" and receive_packet_value != "":
query += "and recv_pkts {0} {1} \n".format(receive_packet_opt,
receive_packet_value)
if session_time_opt != "" and session_time_value != "":
query += "and differ_time {0} {1} \n".format(session_time_opt,
session_time_value)
if session_count_opt != "" and session_count_value != "":
query += "and session_cnt {0} {1} \n".format(session_count_opt,
session_count_value)
if flag_urg_opt != "" and flag_urg_value != "":
query += "and tcp_flags_URG {0} {1} \n".format(flag_urg_opt,
flag_urg_value)
if flag_ack_opt != "" and flag_ack_value != "":
query += "and tcp_flags_ACK {0} {1} \n".format(flag_ack_opt,
flag_ack_value)
if flag_psh_opt != "" and flag_psh_value != "":
query += "and tcp_flags_PSH {0} {1} \n".format(flag_psh_opt,
flag_psh_value)
if flag_rst_opt != "" and flag_rst_value != "":
query += "and tcp_flags_RST {0} {1} \n".format(flag_rst_opt,
flag_rst_value)
if flag_syn_opt != "" and flag_syn_value != "":
query += "and tcp_flags_SYN {0} {1} \n".format(flag_syn_opt,
flag_syn_value)
if flag_fin_opt != "" and flag_fin_value != "":
query += "and tcp_flags_FIN {0} {1} \n".format(flag_fin_opt,
flag_fin_value)
if svr_con_count_opt != "" and svr_con_count_value != "":
query += "and svrip_base_conn_cnt {0} {1} \n".format(
svr_con_count_opt, svr_con_count_value)
if cli_con_count_opt != "" and cli_con_count_value != "":
query += "and clip_base_conn_cnt {0} {1} \n".format(
cli_con_count_opt, cli_con_count_value)
if geo_distance_opt != "" and geo_distance_value != "":
query += "and geoip_distance {0} {1} \n".format(
geo_distance_opt, geo_distance_value)
return query