def statistics(self, keys):
"""
perform data aggregation on the currently acquired data instead of
directly returning the result of the data aggregation of the API.
:param keys: str, user input filter fields
{'app': {'Gunicorn': 2, 'nginx': 14, 'Apache httpd': 9, '[unknown]': 3, 'Tornado httpd': 2}, 'port': {443: 29, 8443: 1}}
:return: None
"""
data = {}
key_list = keys.split(',')
# cycle key
for key in key_list:
count = {}
for item in self.dork_data[:self.num]:
zmdict = ZoomEyeDict(item)
if stat_host_table.get(key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(stat_host_table.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
fields = zmdict.find(stat_host_table.get(key.strip()))
# the value of the result field returned by the API may be empty
if fields == '':
fields = '[unknown]'
r = count.get(fields)
if not r:
count[fields] = 1
else:
count[fields] = count[fields] + 1
data[key] = count
# print result for current data aggregation
show.print_stat(keys, data)
def get_data(self):
"""
get user level and IP historical data
"""
normal_user = ['user', 'developer']
api_key, access_token = file.get_auth_key()
zm = ZoomEye(api_key=api_key, access_token=access_token)
role = zm.resources_info()
# permission restrictions
if role["plan"] in normal_user:
show.printf(
"this function is only open to advanced users and VIP users.",
color='red')
exit(0)
# the user chooses to force data from the API
if self.force:
history_data = zm.history_ip(self.ip)
else:
# from local cache get data
history_data_str = self.get_data_from_cache()
# local cache not exists from API get data
if history_data_str is None:
history_data = zm.history_ip(self.ip)
else:
history_data = json.loads(history_data_str)
# cache data
self.cache_data(history_data)
return history_data
def ip_history(args):
"""
query device history
please see: https://www.zoomeye.org/doc#history-ip-search
:param args:
:return:
"""
ip = args.ip
filters = args.filter
force = args.force
number = args.num
# determine whether the input is an IP address by regular
compile_ip = re.compile(
'^(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|[1-9])\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.'
'(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)$'
)
# IP format error，exit program
if not compile_ip.match(ip):
show.printf("[{}] it is not an IP address, please check!".format(ip),
color='red')
return
zm = HistoryDevice(ip, force, number)
# user input filter field
if filters:
filter_list = filters.split(',')
zm.filter_fields(filter_list)
return
# no filter field,
# print [timestamp,service,port,country,raw_data] fields
zm.show_fields()
def search(args):
dork = args.dork
num = args.num
facet = args.facet
filters = args.filter
stat = args.stat
save = args.save
count_total = args.count
figure = args.figure
force = args.force
resource = args.type
cli_zoom = CliZoomEye(dork, num, resource=resource, facet=facet, force=force)
if filters:
cli_zoom.filter_data(filters, save)
return
if facet:
cli_zoom.facets_data(facet, figure)
return
if count_total:
cli_zoom.count()
return
if stat:
cli_zoom.statistics(stat, figure)
return
if save:
cli_zoom.save(save)
return
if filters is None and facet is None and stat is None:
cli_zoom.default_show()
return
show.printf("please run <zoomeye search -h> for help.")
def filter_search_data(keys, field_table, data):
"""
get the data of the corresponding field
:param keys: list, user input field
:param field_table: dict, fileds
:param data: list, zoomeye api data
:return: list, ex: [[1,2,3...],[1,2,3...],[1,2,3...]...]
"""
result = []
for d in data:
item = []
zmdict = ZoomEyeDict(d)
for key in keys:
if field_table.get(key.strip()) is None:
support_fields = ','.join(list(field_table.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
res = zmdict.find(field_table.get(key.strip()))
if key == "time":
utc_time = datetime.datetime.strptime(res, "%Y-%m-%dT%H:%M:%S")
res = str(utc_time + datetime.timedelta(hours=8))
item.append(res)
result.append(item)
return result
def get_host_item(data):
if data:
ip = data.get('ip')
country = data.get("geoinfo").get("country").get("names").get("en")
port = data.get("portinfo").get("port")
app = data.get("portinfo").get("app")
banner = data.get("portinfo").get("banner")
service = data.get("portinfo").get("service")
return ip, country, port, app, banner, service
else:
show.printf("data cannot be empty", color='red')
exit(0)
def info(args):
"""
used to print the current identity of the user and the remaining data quota for the month
:param args:
:return:
"""
api_key, access_token = file.get_auth_key()
zm = ZoomEye(api_key=api_key, access_token=access_token)
# get user information
user_data = zm.resources_info()
if user_data:
# show in the terminal
show.printf("Role: {}".format(user_data["plan"]))
show.printf("Quota: {}".format(user_data["resources"].get("search")))
def jwt_init(username, password):
"""
initialize through the user name and password, write jwt to the local configuration file,
the expiration time is about 12 hours, so it is recommended to initialize through the api key.
:param username: str, login zoomeye account
:param password: str, login zoomeye account password
:return:
"""
file.check_exist(zoomeye_dir)
try:
zoom = ZoomEye(username=username, password=password)
access_token = zoom.login()
except Exception:
return
jwt_file = zoomeye_dir + "/jwt"
if access_token:
# display the remaining resources of the current account
user_data = zoom.resources_info()
show.printf("Role: {}".format(user_data["plan"]))
show.printf("Quota: {}".format(user_data["resources"].get("search")))
with open(jwt_file, 'w') as f:
f.write(access_token)
show.printf("successfully initialized", color="green")
# change the permission of the configuration file to read-only
os.chmod(jwt_file, 0o600)
else:
show.printf("failed initialized!", color="red")
def load(self):
"""
load local json file,must be save file
"""
with open(self.dork, 'r') as f:
data = f.read()
try:
json_data = json.loads(data)
except json.decoder.JSONDecodeError:
show.printf('json format error', color='red')
exit(0)
self.total = json_data.get("total", 0)
self.dork_data = json_data.get("matches", "")
self.facet_data = json_data.get("facets", "")
self.num = len(self.dork_data)
def search(args):
"""
search for dork functions, support filtering and view detailed data
:param args:
:return:
"""
dork = args.dork
num = args.num
facet = args.facet
filters = args.filter
stat = args.stat
save = args.save
count_total = args.count
figure = args.figure
force = args.force
cli_zoom = CliZoomEye(dork, num, facet=facet, force=force)
# load local zoomeye export file
if os.path.exists(dork):
dork_data, facet_data, total = cli_zoom.load()
# get data by cache file or api
else:
dork_data, facet_data, total = cli_zoom.from_cache_or_api()
# print data total
if count_total:
show.printf(total)
else:
# user saves data to local folder
if save:
cli_zoom.save(save)
return
# print filter
if filters:
cli_zoom.cli_filter(filters)
return
# when facets is not None print facets data
if facet:
show.print_facets(facet, facet_data, total, figure)
return
if stat:
cli_zoom.statistics(stat, figure)
return
# when facets is None print dork data
if filters is None and facet is None and stat is None:
show.print_data(dork_data)
return
# others
show.printf("please run <zoomeye dork -h> for help.")
def cache_data(self, history_data):
"""
save ip history data to local
:param history_data: dict, ip history data
"""
try:
# write data
# if file path not exists
file.write_file(self.cache_path, json.dumps(history_data))
except FileNotFoundError:
# create fold
# retry write to local file
os.makedirs(os.path.expanduser(config.ZOOMEYE_CACHE_PATH))
file.write_file(self.cache_path, json.dumps(history_data))
except Exception as e:
show.printf("unknown error: {}".format(e))
exit(0)
def get_item(data):
"""
take out the corresponding data from the json file.
:param data: dork data ,dict
:return:
"""
if data:
ip = data.get('ip')
country = data.get("geoinfo").get("country").get("names").get("en")
port = data.get("portinfo").get("port")
app = data.get("portinfo").get("app")
banner = data.get("portinfo").get("banner")
service = data.get("portinfo").get("service")
return ip, country, port, app, banner, service
else:
show.printf("data cannot be empty", color='red')
exit(0)
def filter_information(self, filters):
"""
filter IP information, filter like key or key,key=value
"""
info_data = self.request_data()
result_data, has_equal, not_equal = process_filter(
filters, info_data, fields_ip)
if len(result_data) == 0:
return
for item in not_equal:
if fields_ip.get(item.strip()) is None:
support_fields = ','.join(list(fields_ip.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(item, support_fields),
color='red')
exit(0)
show.print_info_filter(not_equal, result_data, has_equal)
def information_ip(args):
ip = args.ip
filters = args.filter
# determine whether the input is an IP address by regular
compile_ip = re.compile('^(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|[1-9])\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.'
'(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)\.(1\d{2}|2[0-4]\d|25[0-5]|[1-9]\d|\d)$')
# IP format error，exit program
if not compile_ip.match(ip):
show.printf("[{}] it is not an IP address, please check!".format(ip), color='red')
return
infor = IPInformation(ip)
if filters:
filter_list = filters.split(',')
infor.filter_information(filter_list)
return
infor.show_information()
def save(self, fields):
"""
save the data to a local json file,
you cannot specify the save path, but you can specify the save data
:param fields: str, filter fields, ex: app=xxxx
:return:
"""
# -save default, data format ex:
# {"total":xxx, "matches":[{...}, {...}, {...}...], "facets":{{...}, {...}...}}
# filter special characters in file names
name = re.findall(r"[a-zA-Z0-9_\u4e00-\u9fa5]+", self.dork)
re_name = '_'.join(name)
if fields == 'all':
filename = "{}_{}_{}.json".format(re_name, self.num,
int(time.time()))
data = {
'total': self.total,
'matches': self.dork_data[:self.num],
'facets': self.facet_data
}
file.write_file(filename, json.dumps(data))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
# -save xx=xxxx, save the filtered data. data format ex:
# {app:"xxx", "app":"httpd",.....}
else:
key, value = self.cli_filter(fields, save=True)
filename = "{}_{}_{}.json".format(re_name, len(value),
int(time.time()))
# parser data
for v in value:
dict_save = {}
for index in range(len(key.split(','))):
dict_key = key.split(',')[index]
dict_value = v[index]
dict_save[dict_key] = dict_value
# write to local file
file.add_to_file(filename, str(dict_save))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
def filter_fields(self, fields):
"""
filter historical IP data
:param fields: list, user input filter fields
"""
has_equal = []
not_equal = []
data = self.drop_web_data()
if not data:
return
# resolve specific filter items
for field_item in fields:
field_split = field_item.split('=')
if len(field_split) == 2:
has_equal.append(field_item)
not_equal.append(field_split[0])
if len(field_split) == 1:
if field_item == "*":
not_equal = list(fields_tables_history_host.keys())
continue
else:
not_equal.append(field_item)
# match filters that contain specific data
if len(has_equal) != 0:
result_data = regexp(has_equal, fields_tables_history_host,
data[:self.num])
else:
result_data = data[:self.num]
# no regexp data
if len(result_data) == 0:
return
# check user input filed is or not support
for item in not_equal:
if fields_tables_history_host.get(item.strip()) is None:
support_fields = ','.join(
list(fields_tables_history_host.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(item, support_fields),
color='red')
exit(0)
show.print_filter_history(not_equal, result_data)
def save(self, fields):
"""
save api response raw data
"""
self.request_data()
name = re.findall(r"[a-zA-Z0-9_\u4e00-\u9fa5]+", self.dork)
re_name = '_'.join(name)
if fields == 'all':
filename = "{}_{}_{}_{}.json".format(re_name,
self.num, self.resource,
int(time.time()))
data = {
'total': self.total,
'matches': self.dork_data,
'facets': self.facet_data
}
with open(filename, 'w') as f:
f.write(json.dumps(data))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
# -save xx=xxxx, save the filtered data. data format ex:
# {app:"xxx", "app":"httpd",.....}
else:
key, value = self.filter_data(fields, save=True)
filename = "{}_{}_{}.json".format(re_name, len(value),
int(time.time()))
# parser data
for v in value:
dict_save = {}
for index in range(len(key.split(','))):
dict_key = key.split(',')[index]
if isinstance(v[index], dict):
v[index] = v[index].get('name', 'unknown')
dict_value = v[index]
dict_save[dict_key] = dict_value
# write to local file
file.add_to_file(filename, str(dict_save))
show.printf("save file to {}/{} successful!".format(
os.getcwd(), filename),
color='green')
def init(args):
"""
the initialization processing function will select the initialization method according to the user's input.
:param args:
:return:
"""
api_key = args.apikey
username = args.username
password = args.password
# use api key init
if api_key and username is None and password is None:
key_init(api_key)
return
# use username and password init
if api_key is None and username and password:
jwt_init(username, password)
return
# invalid parameter
show.printf("input parameter error", color="red")
show.printf("please run <zoomeye init -h> for help.", color="red")
def filter_data(self, keys, data):
"""
get the data of the corresponding field
:param keys: list, user input field
:param data: list, zoomeye api data
:return: list, ex: [[1,2,3...],[1,2,3...],[1,2,3...]...]
"""
result = []
for d in data:
item = []
zmdict = ZoomEyeDict(d)
for key in keys:
if fields_tables_host.get(key.strip()) is None:
support_fields = ','.join(list(fields_tables_host.keys()))
show.printf("filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields), color='red')
exit(0)
res = zmdict.find(fields_tables_host.get(key.strip()))
item.append(res)
result.append(item)
return result
def key_init(key):
"""
initialize through the api key, write the api key to the local configuration file,
theoretically it will never expire unless you remake the api key
:param key: user input API key
:return:
"""
file.check_exist(zoomeye_dir)
key = key.strip()
try:
zoom = ZoomEye(api_key=key)
except Exception:
return
# api key save path
key_file = zoomeye_dir + "/apikey"
# display the remaining resources of the current account
user_data = zoom.resources_info()
show.printf("Role: {}".format(user_data["plan"]))
show.printf("Quota: {}".format(user_data["resources"].get("search")))
# save api key
with open(key_file, 'w') as f:
f.write(key)
show.printf("successfully initialized", color="green")
# change the permission of the configuration file to read-only
os.chmod(key_file, 0o600)
def clear_file(args):
"""
clear user setting and zoomeye cache data
"""
setting = args.setting
cache = args.cache
target_dir = None
# clear user setting
if setting:
target_dir = zoomeye_dir
# clear local cache file
if cache:
target_dir = os.path.expanduser(config.ZOOMEYE_CACHE_PATH)
# user input error
if target_dir is None:
show.printf("Please run <zoomeye clear -h> for help!", color='red')
return
# remove all files under the folder
file_list = os.listdir(target_dir)
for item in file_list:
os.remove(os.path.join(target_dir, item))
show.printf("clear complete!", color='green')
def filter_fields(self, fields):
"""
filter historical IP data
:param fields: list, user input filter fields
"""
data = self.drop_web_data()
result_data, has_equal, not_equal = process_filter(
fields, data, fields_tables_history_host)
# no regexp data
if len(result_data) == 0:
return
# check user input filed is or not support
for item in not_equal:
if fields_tables_history_host.get(item.strip()) is None:
support_fields = ','.join(
list(fields_tables_history_host.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(item, support_fields),
color='red')
exit(0)
show.print_filter_history(not_equal, result_data[:self.num], has_equal)
def handle_page(self):
try:
num = int(self.num)
if num % 20 == 0:
return int(num / 20)
return int(num / 20) + 1
except ValueError:
if self.num == 'all':
for i in range(3):
user_confirm = input(
"The data may exceed your quota. "
"If the quota is exceeded, all quota data will be returned. "
"Are you sure you want to get all the data?(y/N)\n")
if user_confirm == 'y':
self.zoomeye.dork_search(dork=self.dork,
page=1,
resource=self.resource,
facets=self.facet)
self.num = self.zoomeye.total
cache = Cache(self.dork,
page=1,
resource=self.resource)
cache.save(json.dumps(self.zoomeye.raw_data))
if self.num % 20 == 0:
return int(self.num / 20)
return int(self.num / 20) + 1
elif user_confirm == "N":
user_num = input(
"Please enter the required amount of data:")
return int(user_num)
else:
continue
show.printf("more than the number of errors!", color='red')
except Exception as e:
show.printf(e, color='red')
exit(0)
def statistics(self, keys, figure):
"""
local data statistics
"""
self.request_data()
if self.resource == 'web':
tables = stat_web_table
if self.resource == 'host':
tables = stat_host_table
data = {}
key_list = keys.split(',')
# cycle key
for key in key_list:
count = {}
for item in self.dork_data[:self.num]:
zmdict = ZoomEyeDict(item)
if tables.get(key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(tables.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(key, support_fields),
color='red')
exit(0)
fields = zmdict.find(tables.get(key.strip()))
# the value of the result field returned by the API may be empty
if fields == '' or isinstance(fields, list):
fields = '[unknown]'
r = count.get(fields)
if not r:
count[fields] = 1
else:
count[fields] = count[fields] + 1
data[key] = count
# print result for current data aggregation
show.print_stat(keys, data, self.num, figure)
def regexp_data(self, keys):
"""
filter based on fields entered by the user
AND operation on multiple fields
:param keys: str , user input filter filed
:return: list, ex:[{...}, {...}, {...}...]
"""
keys = keys.split(",")
result = []
self.zoomeye.data_list = self.dork_data[:self.num]
data_list = self.zoomeye.data_list
for key in keys:
result = []
for da in data_list:
zmdict = ZoomEyeDict(da)
input_key, input_value = key.split("=")
if fields_tables_host.get(input_key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(fields_tables_host.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(input_key, support_fields),
color='red')
exit(0)
# the value obtained here is of type int, and the user's input is of type str,
# so it needs to be converted.
if input_key == "port":
input_value = str(input_value)
find_value = zmdict.find(
fields_tables_host.get(input_key.strip()))
# get the value through regular matching
try:
regexp_result = re.search(str(input_value),
str(find_value), re.I)
except re.error:
show.printf(
'the regular expression you entered is incorrect, please check!',
color='red')
exit(0)
except Exception as e:
show.printf(e, color='red')
exit(0)
# the matched value is neither None nor empty
if regexp_result and regexp_result.group(0) != '':
result.append(da)
# AND operation
data_list = result
return result
def regexp(keys, field_table, data_list):
"""
match the corresponding data through regular
"""
result = []
for key in keys:
result = []
for da in data_list:
zmdict = ZoomEyeDict(da)
input_key, input_value = key.split("=")
if field_table.get(input_key.strip()) is None:
# check filed effectiveness
support_fields = ','.join(list(field_table.keys()))
show.printf(
"filter command has unsupport fields [{}], support fields has [{}]"
.format(input_key, support_fields),
color='red')
exit(0)
# the value obtained here is of type int, and the user's input is of type str,
# so it needs to be converted.
if input_key == "port":
input_value = str(input_value)
find_value = zmdict.find(field_table.get(input_key.strip()))
# get the value through regular matching
try:
regexp_result = re.search(str(input_value), str(find_value),
re.I)
except re.error:
show.printf(
'the regular expression you entered is incorrect, please check!',
color='red')
exit(0)
except Exception as e:
show.printf(e, color='red')
exit(0)
# the matched value is neither None nor empty
if regexp_result and regexp_result.group(0) != '':
result.append(da)
# AND operation
data_list = result
return result
def count(self):
"""
show dock count number
"""
self.request_data()
show.printf(self.total)