def verify_request(self, flag, type="request"):
"""
校验ceye接口是否有数据
:param flag: 输入的flag
:param type: 请求类型(dns|request),默认是request
:return: Boolean
"""
if not self.check_account():
return False
ret_val = False
counts = 3
url = "http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}".format(
token=self.token, type=type, flag=flag)
while counts:
try:
time.sleep(1)
resp = requests.get(url)
if resp and resp.status_code == 200 and flag in resp.text:
ret_val = True
break
except Exception as ex:
logger.warn(ex)
time.sleep(1)
counts -= 1
return ret_val
def fetch_poc(self, ssvid):
if self.check_account():
try:
if ssvid and ssvid.startswith('ssvid-'):
ssvid = ssvid.split('ssvid-')[-1]
resp = requests.get(
'https://www.seebug.org/api/user/poc_detail?id=%s' % ssvid,
headers=self.headers)
if resp and resp.status_code == 200 and "code" in resp.json():
poc = resp.json()['code']
return poc
elif resp.status_code == 200 and "status" in resp.json(
) and resp.json()["status"] is False:
if "message" in resp.json():
msg = resp.json()["message"]
if msg == "没有权限访问此漏洞":
msg = "No permission to access the vulnerability POC"
else:
msg = "Unknown"
msg = "[PLUGIN] " + msg
raise Exception(msg)
except Exception as ex:
logger.error(str(ex))
else:
return None
def search(self, dork, pages=1, resource='ip,port'):
if resource == 'host':
resource = 'ip,port'
else:
resource = 'host'
search_result = set()
try:
for page in range(1, pages + 1):
url = "https://fofa.so/api/v1/search/all?email={user}&key={token}&qbase64={dork}&fields={resource}&page={page}".format(
user=self.user,
token=self.token,
dork=b64encode(dork.encode()).decode(),
resource=resource,
page=page)
resp = requests.get(url, timeout=80)
if resp and resp.status_code == 200 and "results" in resp.json(
):
content = resp.json()
for match in content['results']:
if resource == "ip,port":
search_result.add("%s:%s" % (match[0], match[1]))
else:
if not match.startswith("https://"):
search_result.add("http://" + match)
else:
search_result.add(match)
else:
logger.error("[PLUGIN] Fofa:{}".format(resp.text))
except Exception as ex:
logger.error(str(ex))
return search_result
def search(self, dork, pages=1, resource='web'):
search_result = set()
if kb.comparison:
kb.comparison.add_dork("Zoomeye", dork)
try:
for page in range(1, pages + 1):
url = "https://api.zoomeye.org/{}/search?query={}&page={}&facet=app,os".format(
resource, urllib.parse.quote(dork), page)
resp = requests.get(url, headers=self.headers)
if resp and resp.status_code == 200 and "matches" in resp.json(
):
content = resp.json()
if resource == 'web':
for match in content["matches"]:
ans = match["site"]
search_result.add(ans)
if kb.comparison:
honeypot = False
if "honeypot" in content or "honeypot_lastupdate" in content:
honeypot = True
kb.comparison.add_ip(ans, "Zoomeye", honeypot)
else:
for match in content['matches']:
ans = match['ip']
if 'portinfo' in match:
ans += ':' + str(match['portinfo']['port'])
search_result.add(ans)
if kb.comparison:
honeypot = False
if "honeypot" in match or "honeypot_lastupdate" in match:
honeypot = True
kb.comparison.add_ip(ans, "Zoomeye", honeypot)
except Exception as ex:
logger.error(str(ex))
return search_result
def exact_request(self, flag, type="request"):
"""
Obtain relevant data by accessing the ceye interface
:param flag: Input flag
:param type: Request type (dns|request), the default is request
:return: Return the acquired data
"""
counts = 3
url = (
"http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}"
).format(token=self.token, type=type, flag=flag)
while counts:
try:
time.sleep(1)
resp = requests.get(url)
if resp and resp.status_code == 200 and flag in resp.text:
data = json.loads(resp.text)
for item in data["data"]:
name = item.get("name", '')
pro = flag
suffix = flag
t = get_middle_text(name, pro, suffix, 0)
if t:
return t
break
except Exception as ex:
logger.warn(ex)
time.sleep(1)
counts -= 1
return False
def get_links(self, url, url_ext=()):
# TODO:
# set base url from base tag or current url
self.base_url = url
url_part = urlparse(url)
self.origin = (url_part.scheme, url_part.netloc)
self.urls = {
'url': set(),
'js': set(),
'img': set()
}
if isinstance(url_ext, str):
url_ext = set(url_ext)
self.url_ext = url_ext
debug_msg = "crawler visiting: {0}".format(url)
logger.debug(debug_msg)
resp = requests.get(url)
content_type = resp.headers.get('content-type', '')
if 'text/html' in content_type:
html = resp.text
self.feed(html)
return self.urls
def verify_request(self, flag, type="request"):
"""
Check whether the ceye interface has data
:param flag: Input flag
:param type: Request type (dns|request), the default is request
:return: Boolean
"""
ret_val = False
counts = 3
url = (
"http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}"
).format(token=self.token, type=type, flag=flag)
while counts:
try:
time.sleep(1)
resp = requests.get(url)
if resp and resp.status_code == 200 and flag in resp.text:
ret_val = True
break
except Exception as ex:
logger.warn(ex)
time.sleep(1)
counts -= 1
return ret_val
def exact_request(self, flag, type="request"):
"""
通过访问ceye接口获取相关数据
:param flag: 输入的flag
:param type: 请求类型(dns|request),默认是request
:return:返回获取的数据
"""
if not self.check_account():
return ""
counts = 3
url = "http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}".format(
token=self.token, type=type, flag=flag)
while counts:
try:
time.sleep(1)
resp = requests.get(url)
if resp and resp.status_code == 200 and flag in resp.text:
data = json.loads(resp.text)
for item in data["data"]:
name = item.get("name", '')
pro = "/" + flag
suffix = flag
t = get_middle_text(name, pro, suffix, 7 + len(flag))
if t:
return t
break
except Exception as ex:
logger.warn(ex)
time.sleep(1)
counts -= 1
return False
def token_is_available(self):
if self.token:
try:
resp = requests.get('https://api.shodan.io/account/profile?key={0}'.format(self.token))
if resp and resp.status_code == 200 and "member" in resp.json():
return True
except Exception as ex:
logger.error(str(ex))
return False
def get_resource_info(self):
try:
resp = requests.get('https://api.shodan.io/account/profile?key={0}'.format(self.token))
if resp and resp.status_code == 200 and 'credits' in resp.json():
content = resp.json()
self.credits = content['credits']
return True
except Exception as ex:
logger.error(str(ex))
return False
def getsubdomain(self):
"""
通过ceye token获取子域名
:return:返回获取的域名
"""
r = requests.get("http://api.ceye.io/v1/identify",
headers=self.headers).json()
suffix = ".ceye.io"
indetify = r["data"]["identify"]
return indetify + suffix
def token_is_available(self):
if self.token:
self.headers['Authorization'] = f'JWT {self.token}'
try:
resp = requests.get('https://www.seebug.org/api/user/poc_list',
headers=self.headers)
if resp and resp.status_code == 200 and "name" in resp.text:
return True
except Exception as ex:
logger.error(str(ex))
return False
def token_is_available(self):
if self.token:
headers = {'Authorization': 'JWT %s' % self.token}
try:
resp = requests.get('https://www.seebug.org/api/user/poc_list', headers = headers)
if resp and resp.status_code == 200 and "id" in resp.json()[0]:
self.headers = headers
return True
except Exception as ex:
logger.error(str(ex))
return False
def search_poc(self, keyword):
if self.check_account():
try:
resp = requests.get('https://www.seebug.org/api/user/poc_list?q=%s' % keyword, headers = self.headers)
if resp and resp.status_code == 200:
pocs = json.loads(resp.text)
return pocs
except Exception as ex:
logger.error(str(ex))
else:
return []
def get_available_pocs(self):
if self.check_account():
try:
resp = requests.get('https://www.seebug.org/api/user/poc_list', headers = self.headers)
if resp and resp.status_code == 200:
pocs = resp.json()
return pocs
except Exception as ex:
logger.error(str(ex))
else:
return []
def token_is_available(self):
if self.token:
headers = {'Authorization': 'JWT %s' % self.token}
try:
resp = requests.get('https://api.zoomeye.org/resources-info', headers=headers)
if resp and resp.status_code == 200 and "plan" in resp.json():
self.headers = headers
return True
except Exception as ex:
logger.error(str(ex))
return False
def token_is_available(self):
if self.token:
headers = {'Authorization': 'JWT %s' % self.token}
headers2 = {'Authorization': self.token}
try:
resp = requests.get('http://api.ceye.io/v1/identify',
headers=headers)
if resp and resp.status_code == 200 and "data" in resp.json():
self.headers = headers
return True
resp = requests.get('http://api.ceye.io/v1/identify',
headers=headers2)
if resp and resp.status_code == 200 and "data" in resp.json():
self.headers = headers2
return True
except Exception as ex:
logger.error(str(ex))
return False
def get_resource_info(self):
if self.check_account():
try:
resp = requests.get('https://api.zoomeye.org/resources-info', headers=self.headers)
if resp and resp.status_code == 200 and 'plan' in resp.json():
content = resp.json()
self.plan = content['plan']
self.resources = content['resources']['search']
return True
except Exception as ex:
logger.error(str(ex))
return False
def token_is_available(self):
if self.token and self.user:
try:
resp = requests.get(
f'{self.api_url}/info/my?email={self.user}&key={self.token}',
headers=self.headers)
logger.info(resp.text)
if resp and resp.status_code == 200 and "username" in resp.json():
return True
except Exception as ex:
logger.error(str(ex))
return False
def token_is_available(self):
if self.token and self.user:
try:
resp = requests.get(
'https://fofa.so/api/v1/info/my?email={user}&key={token}'.
format(user=self.user, token=self.token))
if resp and resp.status_code == 200 and "username" in resp.json(
):
return True
except Exception as ex:
logger.error(str(ex))
return False
def getsubdomain(self):
"""
Obtain subdomains through ceye token
:return: Return the obtained domain name
"""
r = requests.get("http://api.ceye.io/v1/identify",
headers=self.headers).json()
suffix = ".ceye.io"
try:
indetify = r["data"]["identify"]
except KeyError:
return None
return indetify + suffix
def token_is_available(self):
if self.token:
self.headers['Authorization'] = f'JWT {self.token}'
try:
resp = requests.get('https://api.zoomeye.org/resources-info',
headers=self.headers)
if resp and resp.status_code == 200 and "plan" in resp.text:
return True
else:
logger.info(resp.text)
except Exception as ex:
logger.error(str(ex))
return False
def getsubdomain(self):
"""
通过ceye token获取子域名
:return:返回获取的域名
"""
if not self.check_account():
return None
r = requests.get("http://api.ceye.io/v1/identify",
headers=self.headers).json()
suffix = ".ceye.io"
try:
indetify = r["data"]["identify"]
except KeyError:
return None
return indetify + suffix
def fetch_poc(self, ssvid):
if self.check_account():
try:
if ssvid and ssvid.startswith('ssvid-'):
ssvid = ssvid.split('ssvid-')[-1]
resp = requests.get(
'https://www.seebug.org/api/user/poc_detail?id=%s' % ssvid,
headers=self.headers)
if resp and resp.status_code == 200 and "code" in resp.json():
poc = resp.json()['code']
return poc
except Exception as ex:
logger.error(str(ex))
else:
return None
def token_is_available(self):
if self.token:
# distinguish Jwt Token & API Token
self.headers['Authorization'] = self.token if len(
self.token) < 48 else f'JWT {self.token}'
try:
resp = requests.get('http://api.ceye.io/v1/identify',
headers=self.headers)
if resp and resp.status_code == 200 and "identify" in resp.text:
return True
else:
logger.info(resp.text)
except Exception as ex:
logger.error(str(ex))
return False
def token_is_available(self):
if self.token:
try:
headers = {
"X-QuakeToken": self.token,
"Content-Type": "application/json"
}
resp = requests.get('https://quake.360.cn/api/v3/user/info',
headers=headers)
if resp and resp.status_code == 200 and resp.json(
)['code'] == 0:
return True
except Exception as ex:
logger.error(str(ex))
return False
def token_is_available(self):
if self.token:
try:
self.headers['X-QuakeToken'] = self.token
resp = requests.get('https://quake.360.cn/api/v3/user/info',
headers=self.headers)
if 'month_remaining_credit' not in resp.text:
logger.info(resp.text)
if resp and resp.status_code == 200 and resp.json(
)['code'] == 0:
return True
except Exception as ex:
logger.error(str(ex))
return False
def get_redirect_url(url):
# TODO:
# regex need more test cases
meta_regex = '(?is)\<meta[^<>]*?url\s*=([\d\w://\\\\.?=&;%-]*)[^<>]*'
body_regex = '''(?is)\<body[^<>]*?location[\s\.\w]*=['"]?([\d\w://\\\\.?=&;%-]*)['"]?[^<>]*'''
js_regex = '''(?is)<script.*?>[^<>]*?location\.(?:replace|href|assign)[=\("']*([\d\w://\\\\.?=&;%-]*)[^<>]*?</script>'''
resp = requests.get(url)
true_url = resp.url
for regex in [meta_regex, body_regex, js_regex]:
result = re.search(regex, resp.text)
if result:
redirect_url = result.group(1)
true_url = urljoin(url, redirect_url)
break
return true_url
def search(self, dork, pages=1, resource='host'):
search_result = set()
try:
for page in range(1, pages + 1):
url = "https://api.shodan.io/shodan/{0}/search?key={1}&query={2}&page={3}".format(resource,
self.token,
urllib.parse.quote(dork),
page)
resp = requests.get(url)
if resp and resp.status_code == 200 and "total" in resp.json():
content = resp.json()
for match in content['matches']:
ans = match['ip_str']
if 'port' in match:
ans += ':' + str(match['port'])
search_result.add(ans)
except Exception as ex:
logger.error(str(ex))
return search_result
def search(self, dork, pages=1, resource='web'):
search_result = set()
try:
for page in range(1, pages + 1):
url = "https://api.zoomeye.org/{}/search?query={}&page={}&facet=app,os".format(
resource, urllib.parse.quote(dork), page)
resp = requests.get(url, headers=self.headers)
if resp and resp.status_code == 200 and "matches" in resp.json(
):
content = resp.json()
if resource == 'web':
search_result.update(
[match['site'] for match in content['matches']])
else:
for match in content['matches']:
ans = match['ip']
if 'portinfo' in match:
ans += ':' + str(match['portinfo']['port'])
search_result.add(ans)
except Exception as ex:
logger.error(str(ex))
return search_result