def request_data(self):
"""
get api data
"""
api_key, access_token = file.get_auth_key()
zm = ZoomEye(api_key=api_key, access_token=access_token)
data = zm.dork_search(self.dork)
return data
def zoomeye_engine(api_key, ip, count):
zm = ZoomEye(api_key=api_key) #API-KEY for 認證
pattern = "cidr:" + ip
data = zm.dork_search(pattern)
#print(data)
num = 1
for datum in data:
if (num > count):
return
format(datum)
num += 1
class CliZoomEye:
"""
the ZoomEye instance in cli mode has added storage compared to
the ZoomEye instance of the API, to determine where
to get the data, cache or api.
By the way, cli mode supports search for "host", but "web" search is not currently supported
"""
def __init__(self, dork, num, facet=None):
self.dork = dork
self.num = num
self.facet = facet
self.dork_data = []
self.facet_data = None
self.total = 0
self.api_key, self.access_token = file.get_auth_key()
self.zoomeye = ZoomEye(api_key=self.api_key,
access_token=self.access_token)
def handle_page(self):
"""
convert the number of data into pages and round up.
ex: num = 30, page = 2.
because the API returns 20 pieces of data each time.
:return:
"""
num = int(self.num)
if num % 20 == 0:
return int(num / 20)
return int(num / 20) + 1
def auto_cache(self, data, page):
"""
cache to local
:param page:
:param data:
:return:
"""
cache = Cache(self.dork, page)
cache.save(json.dumps(data))
def from_cache_or_api(self):
"""
get data from cache or api
get data from the api if there is no data in the cache.
:return:
"""
page = self.handle_page()
for p in range(page):
cache = Cache(self.dork, page=p)
# get data from cache file
if cache.check():
# return dork, facet, total data
dork_data_list, self.facet_data, self.total = cache.load()
self.dork_data.extend(dork_data_list)
else:
# no cache, get data from API
self.facet = [
'app', 'device', 'service', 'os', 'port', 'country', 'city'
]
try:
dork_data_list = self.zoomeye.dork_search(
dork=self.dork,
page=p + 1,
resource="host",
facets=self.facet)
except ValueError:
print(
"the access token expires, please re-run [zoomeye init] command. "
"it is recommended to use API KEY for initialization!")
exit(0)
self.facet_data = self.zoomeye.facet_data
self.total = self.zoomeye.total
self.dork_data.extend(dork_data_list)
if dork_data_list:
self.auto_cache(self.zoomeye.raw_data, p)
# return dork, facet,total data
return self.dork_data[:self.num], self.facet_data, self.total
def filter_data(self, keys, data):
"""
get the data of the corresponding field
:param keys: list, user input field
:param data: list, zoomeye api data
:return: list, ex: [[1,2,3...],[1,2,3...],[1,2,3...]...]
"""
result = []
for d in data:
item = []
zmdict = ZoomEyeDict(d)
for key in keys:
if fields_tables_host.get(key.strip()) is None:
support_fields = ','.join(list(fields_tables_host.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
res = zmdict.find(fields_tables_host.get(key.strip()))
item.append(res)
result.append(item)
return result
def regexp_data(self, keys):
"""
filter based on fields entered by the user
AND operation on multiple fields
:param keys: str , user input filter filed
:return: list, ex:[{...}, {...}, {...}...]
"""
keys = keys.split(",")
result = []
self.zoomeye.data_list = self.dork_data[:self.num]
data_list = self.zoomeye.data_list
for key in keys:
result = []
for da in data_list:
zmdict = ZoomEyeDict(da)
input_key, input_value = key.split("=")
if fields_tables_host.get(input_key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(fields_tables_host.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(input_key, support_fields),
color='red')
exit(0)
# the value obtained here is of type int, and the user's input is of type str,
# so it needs to be converted.
if input_key == "port":
input_value = str(input_value)
find_value = zmdict.find(
fields_tables_host.get(input_key.strip()))
# get the value through regular matching
try:
regexp_result = re.search(str(input_value),
str(find_value), re.I)
except re.error:
show.printf(
'the regular expression you entered is incorrect, please check!',
color='red')
exit(0)
except Exception as e:
show.printf(e, color='red')
exit(0)
# the matched value is neither None nor empty
if regexp_result and regexp_result.group(0) != '':
result.append(da)
# AND operation
data_list = result
return result
def cli_filter(self, keys, save=False):
"""
this function is used to filter the results.
:param save:
:param keys: str, filter condition. ex: 'ip, port, app=xxx'
:return: None
"""
has_equal = []
not_equal = []
# set the ip field to be displayed by default and in the first place
key_list = keys.split(',')
try:
# set ip field in the first place
key_index = key_list.index("ip")
key_list.pop(key_index)
key_list.insert(0, 'ip')
# add IP to the first item when there is no IP
except ValueError:
key_list.insert(0, 'ip')
# process user input fields, separating single fields from fields with equal signs.
for key in key_list:
res = key.split('=')
# field with equal sign
if len(res) == 2:
has_equal.append(key)
not_equal.append(res[0])
# No field with equal sign
if len(res) == 1:
# handle the case where the wildcard character * is included in the filed
# that is, query all fields
if key == "*":
not_equal = list(fields_tables_host.keys())
continue
else:
not_equal.append(key)
# the filter condition is port, banner, app=**
# ex:port,banner,app=MySQL
if len(has_equal) != 0:
equal = ','.join(has_equal)
equal_data = self.regexp_data(equal)
# the filter condition is app, port
# ex: ip,port,app
else:
equal_data = self.dork_data[:self.num]
# get result
result = self.filter_data(not_equal, equal_data)
equal = ','.join(not_equal)
if save:
return equal, result
show.print_filter(equal, result)
def save(self, fields):
"""
save the data to a local json file,
you cannot specify the save path, but you can specify the save data
:param fields: str, filter fields, ex: app=xxxx
:return:
"""
# -save default, data format ex:
# {"total":xxx, "matches":[{...}, {...}, {...}...], "facets":{{...}, {...}...}}
if fields == 'all':
filename = "{}_{}_{}.json".format(self.dork, self.num,
int(time.time()))
data = {
'total': self.total,
'matches': self.dork_data[:self.num],
'facets': self.facet_data
}
file.write_file(filename, json.dumps(data))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
# -save xx=xxxx, save the filtered data. data format ex:
# {app:"xxx", "app":"httpd",.....}
else:
key, value = self.cli_filter(fields, save=True)
filename = "{}_{}_{}.json".format(self.dork, len(value),
int(time.time()))
# parser data
for v in value:
dict_save = {}
for index in range(len(key.split(','))):
dict_key = key.split(',')[index]
dict_value = v[index]
dict_save[dict_key] = dict_value
# write to local file
file.add_to_file(filename, str(dict_save))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
def load(self):
"""
load a local file
it must be a json file and the data format is the format exported by zoomeye.
format is {"total":123123, "matches":[{...}, {...}, {...}...], "facets":{{...}, {...}...}}
:param path:
:return:
"""
data = file.read_file(self.dork)
json_data = json.loads(data)
self.total = json_data.get("total", 0)
self.dork_data = json_data.get("matches", "")
self.facet_data = json_data.get("facets", "")
if self.total == 0 and self.dork_data == "" and self.facet_data:
print("file format error!")
exit(0)
self.num = len(self.dork_data)
return self.dork_data, self.facet_data, self.total
def statistics(self, keys):
"""
perform data aggregation on the currently acquired data instead of
directly returning the result of the data aggregation of the API.
:param keys: str, user input filter fields
{'app': {'Gunicorn': 2, 'nginx': 14, 'Apache httpd': 9, '[unknown]': 3, 'Tornado httpd': 2}, 'port': {443: 29, 8443: 1}}
:return: None
"""
data = {}
key_list = keys.split(',')
# cycle key
for key in key_list:
count = {}
for item in self.dork_data[:self.num]:
zmdict = ZoomEyeDict(item)
if stat_host_table.get(key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(stat_host_table.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
fields = zmdict.find(stat_host_table.get(key.strip()))
# the value of the result field returned by the API may be empty
if fields == '':
fields = '[unknown]'
r = count.get(fields)
if not r:
count[fields] = 1
else:
count[fields] = count[fields] + 1
data[key] = count
# print result for current data aggregation
show.print_stat(keys, data)
class CliZoomEye:
def __init__(self, dork, num, resource, facet=None, force=False):
self.dork = dork
self.num = num
self.resource = resource
self.facet = facet
self.force = force
self.dork_data = list()
self.facet_data = None
self.total = 0
self.api_key, self.access_token = file.get_auth_key()
self.zoomeye = ZoomEye(api_key=self.api_key,
access_token=self.access_token)
def handle_page(self):
try:
num = int(self.num)
if num % 20 == 0:
return int(num / 20)
return int(num / 20) + 1
except ValueError:
if self.num == 'all':
for i in range(3):
user_confirm = input(
"The data may exceed your quota. "
"If the quota is exceeded, all quota data will be returned. "
"Are you sure you want to get all the data?(y/N)\n")
if user_confirm == 'y':
self.zoomeye.dork_search(dork=self.dork,
page=1,
resource=self.resource,
facets=self.facet)
self.num = self.zoomeye.total
cache = Cache(self.dork,
page=1,
resource=self.resource)
cache.save(json.dumps(self.zoomeye.raw_data))
if self.num % 20 == 0:
return int(self.num / 20)
return int(self.num / 20) + 1
elif user_confirm == "N":
user_num = input(
"Please enter the required amount of data:")
return int(user_num)
else:
continue
show.printf("more than the number of errors!", color='red')
except Exception as e:
show.printf(e, color='red')
exit(0)
def cache_dork(self, page, data):
cache_file = Cache(self.dork, page=page, resource=self.resource)
cache_file.save(json.dumps(data))
def request_data(self):
if os.path.exists(self.dork):
self.load()
else:
page_count = self.handle_page()
for page in range(page_count):
cache_file = Cache(self.dork, self.resource, page)
if cache_file.check() and self.force is False:
dork_data_list, self.facet_data, self.total = cache_file.load(
)
self.dork_data.extend(dork_data_list)
else:
if self.resource == 'host':
self.facet = [
'app', 'device', 'service', 'os', 'port',
'country', 'city'
]
if self.resource == 'web':
self.facet = [
'webapp', 'component', 'framework', 'frontend',
'server', 'waf', 'os', 'country', 'city'
]
try:
dork_data_list = self.zoomeye.dork_search(
dork=self.dork,
page=page + 1,
resource=self.resource,
facets=self.facet)
except ValueError:
print(
"the access token expires, please re-run [zoomeye init] command."
"it is recommended to use API KEY for initialization!"
)
exit(0)
self.facet_data = self.zoomeye.facet_data
self.total = self.zoomeye.total
self.dork_data.extend(dork_data_list)
self.cache_dork(page, self.zoomeye.raw_data)
def default_show(self):
self.request_data()
if self.resource == 'host':
show.show_host_default_data(self.dork_data, self.total)
if self.resource == 'web':
show.show_web_default_data(self.dork_data, self.total)
def filter_data(self, keys, save=False):
"""
according to web/search or host/search select filter field
"""
if save is not True:
self.request_data()
if self.resource == 'host':
tables = fields_tables_host
if self.resource == 'web':
tables = fields_tables_web
has_equal = []
not_equal = []
# set the ip field to be displayed by default and in the first place
key_list = keys.split(',')
try:
# set ip field in the first place
key_index = key_list.index("ip")
key_list.pop(key_index)
key_list.insert(0, 'ip')
# add IP to the first item when there is no IP
except ValueError:
key_list.insert(0, 'ip')
# process user input fields, separating single fields from fields with equal signs.
for key in key_list:
res = key.split('=')
# field with equal sign
if len(res) == 2:
has_equal.append(key)
# not_equal.append(res[0])
# No field with equal sign
if len(res) == 1:
# handle the case where the wildcard character * is included in the filed
# that is, query all fields
if key == "*":
not_equal = list(tables.keys())
continue
else:
not_equal.append(key)
# the filter condition is port, banner, app=**
# ex:port,banner,app=MySQL
if len(has_equal) != 0:
equal_data = regexp(has_equal, tables, self.dork_data)
# the filter condition is app, port
# ex: ip,port,app
else:
equal_data = self.dork_data
# get result
result = filter_search_data(not_equal, tables, equal_data)
equal = ','.join(not_equal)
if save:
return equal, result
show.print_filter(equal, result[:self.num], has_equal)
def facets_data(self, facet, figure):
"""
according to web/search or host/search select facet field
"""
self.request_data()
if self.resource == 'host':
tables = facets_table_host
if self.resource == 'web':
tables = facets_table_web
show.print_facets(facet, self.facet_data, self.total, figure, tables)
def save(self, fields):
"""
save api response raw data
"""
self.request_data()
name = re.findall(r"[a-zA-Z0-9_\u4e00-\u9fa5]+", self.dork)
re_name = '_'.join(name)
if fields == 'all':
filename = "{}_{}_{}_{}.json".format(re_name,
self.num, self.resource,
int(time.time()))
data = {
'total': self.total,
'matches': self.dork_data,
'facets': self.facet_data
}
with open(filename, 'w') as f:
f.write(json.dumps(data))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
# -save xx=xxxx, save the filtered data. data format ex:
# {app:"xxx", "app":"httpd",.....}
else:
key, value = self.filter_data(fields, save=True)
filename = "{}_{}_{}.json".format(re_name, len(value),
int(time.time()))
# parser data
for v in value:
dict_save = {}
for index in range(len(key.split(','))):
dict_key = key.split(',')[index]
if isinstance(v[index], dict):
v[index] = v[index].get('name', 'unknown')
dict_value = v[index]
dict_save[dict_key] = dict_value
# write to local file
file.add_to_file(filename, str(dict_save))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
def load(self):
"""
load local json file,must be save file
"""
with open(self.dork, 'r') as f:
data = f.read()
try:
json_data = json.loads(data)
except json.decoder.JSONDecodeError:
show.printf('json format error', color='red')
exit(0)
self.total = json_data.get("total", 0)
self.dork_data = json_data.get("matches", "")
self.facet_data = json_data.get("facets", "")
self.num = len(self.dork_data)
def statistics(self, keys, figure):
"""
local data statistics
"""
self.request_data()
if self.resource == 'web':
tables = stat_web_table
if self.resource == 'host':
tables = stat_host_table
data = {}
key_list = keys.split(',')
# cycle key
for key in key_list:
count = {}
for item in self.dork_data[:self.num]:
zmdict = ZoomEyeDict(item)
if tables.get(key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(tables.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
fields = zmdict.find(tables.get(key.strip()))
# the value of the result field returned by the API may be empty
if fields == '' or isinstance(fields, list):
fields = '[unknown]'
r = count.get(fields)
if not r:
count[fields] = 1
else:
count[fields] = count[fields] + 1
data[key] = count
# print result for current data aggregation
show.print_stat(keys, data, self.num, figure)
def count(self):
"""
show dock count number
"""
self.request_data()
show.printf(self.total)