def filter_search_data(keys, field_table, data):
"""
get the data of the corresponding field
:param keys: list, user input field
:param field_table: dict, fileds
:param data: list, zoomeye api data
:return: list, ex: [[1,2,3...],[1,2,3...],[1,2,3...]...]
"""
result = []
for d in data:
item = []
zmdict = ZoomEyeDict(d)
for key in keys:
if field_table.get(key.strip()) is None:
support_fields = ','.join(list(field_table.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
res = zmdict.find(field_table.get(key.strip()))
if key == "time":
utc_time = datetime.datetime.strptime(res, "%Y-%m-%dT%H:%M:%S")
res = str(utc_time + datetime.timedelta(hours=8))
item.append(res)
result.append(item)
return result
def statistics(self, keys):
"""
perform data aggregation on the currently acquired data instead of
directly returning the result of the data aggregation of the API.
:param keys: str, user input filter fields
{'app': {'Gunicorn': 2, 'nginx': 14, 'Apache httpd': 9, '[unknown]': 3, 'Tornado httpd': 2}, 'port': {443: 29, 8443: 1}}
:return: None
"""
data = {}
key_list = keys.split(',')
# cycle key
for key in key_list:
count = {}
for item in self.dork_data[:self.num]:
zmdict = ZoomEyeDict(item)
if stat_host_table.get(key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(stat_host_table.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
fields = zmdict.find(stat_host_table.get(key.strip()))
# the value of the result field returned by the API may be empty
if fields == '':
fields = '[unknown]'
r = count.get(fields)
if not r:
count[fields] = 1
else:
count[fields] = count[fields] + 1
data[key] = count
# print result for current data aggregation
show.print_stat(keys, data)
def regexp_data(self, keys):
"""
filter based on fields entered by the user
AND operation on multiple fields
:param keys: str , user input filter filed
:return: list, ex:[{...}, {...}, {...}...]
"""
keys = keys.split(",")
result = []
self.zoomeye.data_list = self.dork_data[:self.num]
data_list = self.zoomeye.data_list
for key in keys:
result = []
for da in data_list:
zmdict = ZoomEyeDict(da)
input_key, input_value = key.split("=")
if fields_tables_host.get(input_key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(fields_tables_host.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(input_key, support_fields),
color='red')
exit(0)
# the value obtained here is of type int, and the user's input is of type str,
# so it needs to be converted.
if input_key == "port":
input_value = str(input_value)
find_value = zmdict.find(
fields_tables_host.get(input_key.strip()))
# get the value through regular matching
try:
regexp_result = re.search(str(input_value),
str(find_value), re.I)
except re.error:
show.printf(
'the regular expression you entered is incorrect, please check!',
color='red')
exit(0)
except Exception as e:
show.printf(e, color='red')
exit(0)
# the matched value is neither None nor empty
if regexp_result and regexp_result.group(0) != '':
result.append(da)
# AND operation
data_list = result
return result
def regexp(keys, field_table, data_list):
"""
match the corresponding data through regular
"""
result = []
for key in keys:
result = []
for da in data_list:
zmdict = ZoomEyeDict(da)
input_key, input_value = key.split("=")
if field_table.get(input_key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(field_table.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(input_key, support_fields),
color='red')
exit(0)
# the value obtained here is of type int, and the user's input is of type str,
# so it needs to be converted.
if input_key == "port":
input_value = str(input_value)
find_value = zmdict.find(field_table.get(input_key.strip()))
# get the value through regular matching
try:
regexp_result = re.search(str(input_value), str(find_value),
re.I)
except re.error:
show.printf(
'the regular expression you entered is incorrect, please check!',
color='red')
exit(0)
except Exception as e:
show.printf(e, color='red')
exit(0)
# the matched value is neither None nor empty
if regexp_result and regexp_result.group(0) != '':
result.append(da)
# AND operation
data_list = result
return result
def filter_data(self, keys, data):
"""
get the data of the corresponding field
:param keys: list, user input field
:param data: list, zoomeye api data
:return: list, ex: [[1,2,3...],[1,2,3...],[1,2,3...]...]
"""
result = []
for d in data:
item = []
zmdict = ZoomEyeDict(d)
for key in keys:
if fields_tables_host.get(key.strip()) is None:
support_fields = ','.join(list(fields_tables_host.keys()))
show.printf("filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields), color='red')
exit(0)
res = zmdict.find(fields_tables_host.get(key.strip()))
item.append(res)
result.append(item)
return result
def filter_history_data(fileds, host_data, omit=True):
"""
filter historical data based on user input
:param fileds: list, user input
:param host_data: list, exclude web data
:param omit: bool, omit string flag
return: all_data,list matched data
return: port_count, set, count open ports non-repeating
"""
all_data = []
port_count = set()
for host_item in host_data:
every_item = []
host_dict = ZoomEyeDict(host_item)
for filed_item in fileds:
host_result = host_dict.find(
fields_tables_history_host.get(filed_item.strip()))
# count ports
if filed_item == 'port':
port_count.add(host_result)
# format timestamp
if filed_item == 'time':
utc_time = datetime.datetime.strptime(host_result[:19],
"%Y-%m-%dT%H:%M:%S")
host_result = str(utc_time)
# omit raw data, is too long
if filed_item == 'raw':
if omit:
host_result = show.omit_str(show.convert_str(host_result))
else:
host_result = show.convert_str(host_result)
# replace None --> [unknown]
if host_result is None:
host_result = "[unknown]"
every_item.append(host_result)
all_data.append(every_item)
return all_data, port_count
def statistics(self, keys, figure):
"""
local data statistics
"""
self.request_data()
if self.resource == 'web':
tables = stat_web_table
if self.resource == 'host':
tables = stat_host_table
data = {}
key_list = keys.split(',')
# cycle key
for key in key_list:
count = {}
for item in self.dork_data[:self.num]:
zmdict = ZoomEyeDict(item)
if tables.get(key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(tables.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
fields = zmdict.find(tables.get(key.strip()))
# the value of the result field returned by the API may be empty
if fields == '' or isinstance(fields, list):
fields = '[unknown]'
r = count.get(fields)
if not r:
count[fields] = 1
else:
count[fields] = count[fields] + 1
data[key] = count
# print result for current data aggregation
show.print_stat(keys, data, self.num, figure)