def verify_request(self, flag, type="request"): """ 校验ceye接口是否有数据 :param flag: 输入的flag :param type: 请求类型(dns|request),默认是request :return: Boolean """ if not self.check_account(): return False ret_val = False counts = 3 url = "http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}".format( token=self.token, type=type, flag=flag) while counts: try: time.sleep(1) resp = requests.get(url) if resp and resp.status_code == 200 and flag in resp.text: ret_val = True break except Exception as ex: logger.warn(ex) time.sleep(1) counts -= 1 return ret_val
def fetch_poc(self, ssvid): if self.check_account(): try: if ssvid and ssvid.startswith('ssvid-'): ssvid = ssvid.split('ssvid-')[-1] resp = requests.get( 'https://www.seebug.org/api/user/poc_detail?id=%s' % ssvid, headers=self.headers) if resp and resp.status_code == 200 and "code" in resp.json(): poc = resp.json()['code'] return poc elif resp.status_code == 200 and "status" in resp.json( ) and resp.json()["status"] is False: if "message" in resp.json(): msg = resp.json()["message"] if msg == "没有权限访问此漏洞": msg = "No permission to access the vulnerability POC" else: msg = "Unknown" msg = "[PLUGIN] " + msg raise Exception(msg) except Exception as ex: logger.error(str(ex)) else: return None
def search(self, dork, pages=1, resource='ip,port'): if resource == 'host': resource = 'ip,port' else: resource = 'host' search_result = set() try: for page in range(1, pages + 1): url = "https://fofa.so/api/v1/search/all?email={user}&key={token}&qbase64={dork}&fields={resource}&page={page}".format( user=self.user, token=self.token, dork=b64encode(dork.encode()).decode(), resource=resource, page=page) resp = requests.get(url, timeout=80) if resp and resp.status_code == 200 and "results" in resp.json( ): content = resp.json() for match in content['results']: if resource == "ip,port": search_result.add("%s:%s" % (match[0], match[1])) else: if not match.startswith("https://"): search_result.add("http://" + match) else: search_result.add(match) else: logger.error("[PLUGIN] Fofa:{}".format(resp.text)) except Exception as ex: logger.error(str(ex)) return search_result
def search(self, dork, pages=1, resource='web'): search_result = set() if kb.comparison: kb.comparison.add_dork("Zoomeye", dork) try: for page in range(1, pages + 1): url = "https://api.zoomeye.org/{}/search?query={}&page={}&facet=app,os".format( resource, urllib.parse.quote(dork), page) resp = requests.get(url, headers=self.headers) if resp and resp.status_code == 200 and "matches" in resp.json( ): content = resp.json() if resource == 'web': for match in content["matches"]: ans = match["site"] search_result.add(ans) if kb.comparison: honeypot = False if "honeypot" in content or "honeypot_lastupdate" in content: honeypot = True kb.comparison.add_ip(ans, "Zoomeye", honeypot) else: for match in content['matches']: ans = match['ip'] if 'portinfo' in match: ans += ':' + str(match['portinfo']['port']) search_result.add(ans) if kb.comparison: honeypot = False if "honeypot" in match or "honeypot_lastupdate" in match: honeypot = True kb.comparison.add_ip(ans, "Zoomeye", honeypot) except Exception as ex: logger.error(str(ex)) return search_result
def exact_request(self, flag, type="request"): """ Obtain relevant data by accessing the ceye interface :param flag: Input flag :param type: Request type (dns|request), the default is request :return: Return the acquired data """ counts = 3 url = ( "http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}" ).format(token=self.token, type=type, flag=flag) while counts: try: time.sleep(1) resp = requests.get(url) if resp and resp.status_code == 200 and flag in resp.text: data = json.loads(resp.text) for item in data["data"]: name = item.get("name", '') pro = flag suffix = flag t = get_middle_text(name, pro, suffix, 0) if t: return t break except Exception as ex: logger.warn(ex) time.sleep(1) counts -= 1 return False
def get_links(self, url, url_ext=()): # TODO: # set base url from base tag or current url self.base_url = url url_part = urlparse(url) self.origin = (url_part.scheme, url_part.netloc) self.urls = { 'url': set(), 'js': set(), 'img': set() } if isinstance(url_ext, str): url_ext = set(url_ext) self.url_ext = url_ext debug_msg = "crawler visiting: {0}".format(url) logger.debug(debug_msg) resp = requests.get(url) content_type = resp.headers.get('content-type', '') if 'text/html' in content_type: html = resp.text self.feed(html) return self.urls
def verify_request(self, flag, type="request"): """ Check whether the ceye interface has data :param flag: Input flag :param type: Request type (dns|request), the default is request :return: Boolean """ ret_val = False counts = 3 url = ( "http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}" ).format(token=self.token, type=type, flag=flag) while counts: try: time.sleep(1) resp = requests.get(url) if resp and resp.status_code == 200 and flag in resp.text: ret_val = True break except Exception as ex: logger.warn(ex) time.sleep(1) counts -= 1 return ret_val
def exact_request(self, flag, type="request"): """ 通过访问ceye接口获取相关数据 :param flag: 输入的flag :param type: 请求类型(dns|request),默认是request :return:返回获取的数据 """ if not self.check_account(): return "" counts = 3 url = "http://api.ceye.io/v1/records?token={token}&type={type}&filter={flag}".format( token=self.token, type=type, flag=flag) while counts: try: time.sleep(1) resp = requests.get(url) if resp and resp.status_code == 200 and flag in resp.text: data = json.loads(resp.text) for item in data["data"]: name = item.get("name", '') pro = "/" + flag suffix = flag t = get_middle_text(name, pro, suffix, 7 + len(flag)) if t: return t break except Exception as ex: logger.warn(ex) time.sleep(1) counts -= 1 return False
def token_is_available(self): if self.token: try: resp = requests.get('https://api.shodan.io/account/profile?key={0}'.format(self.token)) if resp and resp.status_code == 200 and "member" in resp.json(): return True except Exception as ex: logger.error(str(ex)) return False
def get_resource_info(self): try: resp = requests.get('https://api.shodan.io/account/profile?key={0}'.format(self.token)) if resp and resp.status_code == 200 and 'credits' in resp.json(): content = resp.json() self.credits = content['credits'] return True except Exception as ex: logger.error(str(ex)) return False
def getsubdomain(self): """ 通过ceye token获取子域名 :return:返回获取的域名 """ r = requests.get("http://api.ceye.io/v1/identify", headers=self.headers).json() suffix = ".ceye.io" indetify = r["data"]["identify"] return indetify + suffix
def token_is_available(self): if self.token: self.headers['Authorization'] = f'JWT {self.token}' try: resp = requests.get('https://www.seebug.org/api/user/poc_list', headers=self.headers) if resp and resp.status_code == 200 and "name" in resp.text: return True except Exception as ex: logger.error(str(ex)) return False
def token_is_available(self): if self.token: headers = {'Authorization': 'JWT %s' % self.token} try: resp = requests.get('https://www.seebug.org/api/user/poc_list', headers = headers) if resp and resp.status_code == 200 and "id" in resp.json()[0]: self.headers = headers return True except Exception as ex: logger.error(str(ex)) return False
def search_poc(self, keyword): if self.check_account(): try: resp = requests.get('https://www.seebug.org/api/user/poc_list?q=%s' % keyword, headers = self.headers) if resp and resp.status_code == 200: pocs = json.loads(resp.text) return pocs except Exception as ex: logger.error(str(ex)) else: return []
def get_available_pocs(self): if self.check_account(): try: resp = requests.get('https://www.seebug.org/api/user/poc_list', headers = self.headers) if resp and resp.status_code == 200: pocs = resp.json() return pocs except Exception as ex: logger.error(str(ex)) else: return []
def token_is_available(self): if self.token: headers = {'Authorization': 'JWT %s' % self.token} try: resp = requests.get('https://api.zoomeye.org/resources-info', headers=headers) if resp and resp.status_code == 200 and "plan" in resp.json(): self.headers = headers return True except Exception as ex: logger.error(str(ex)) return False
def token_is_available(self): if self.token: headers = {'Authorization': 'JWT %s' % self.token} headers2 = {'Authorization': self.token} try: resp = requests.get('http://api.ceye.io/v1/identify', headers=headers) if resp and resp.status_code == 200 and "data" in resp.json(): self.headers = headers return True resp = requests.get('http://api.ceye.io/v1/identify', headers=headers2) if resp and resp.status_code == 200 and "data" in resp.json(): self.headers = headers2 return True except Exception as ex: logger.error(str(ex)) return False
def get_resource_info(self): if self.check_account(): try: resp = requests.get('https://api.zoomeye.org/resources-info', headers=self.headers) if resp and resp.status_code == 200 and 'plan' in resp.json(): content = resp.json() self.plan = content['plan'] self.resources = content['resources']['search'] return True except Exception as ex: logger.error(str(ex)) return False
def token_is_available(self): if self.token and self.user: try: resp = requests.get( f'{self.api_url}/info/my?email={self.user}&key={self.token}', headers=self.headers) logger.info(resp.text) if resp and resp.status_code == 200 and "username" in resp.json(): return True except Exception as ex: logger.error(str(ex)) return False
def token_is_available(self): if self.token and self.user: try: resp = requests.get( 'https://fofa.so/api/v1/info/my?email={user}&key={token}'. format(user=self.user, token=self.token)) if resp and resp.status_code == 200 and "username" in resp.json( ): return True except Exception as ex: logger.error(str(ex)) return False
def getsubdomain(self): """ Obtain subdomains through ceye token :return: Return the obtained domain name """ r = requests.get("http://api.ceye.io/v1/identify", headers=self.headers).json() suffix = ".ceye.io" try: indetify = r["data"]["identify"] except KeyError: return None return indetify + suffix
def token_is_available(self): if self.token: self.headers['Authorization'] = f'JWT {self.token}' try: resp = requests.get('https://api.zoomeye.org/resources-info', headers=self.headers) if resp and resp.status_code == 200 and "plan" in resp.text: return True else: logger.info(resp.text) except Exception as ex: logger.error(str(ex)) return False
def getsubdomain(self): """ 通过ceye token获取子域名 :return:返回获取的域名 """ if not self.check_account(): return None r = requests.get("http://api.ceye.io/v1/identify", headers=self.headers).json() suffix = ".ceye.io" try: indetify = r["data"]["identify"] except KeyError: return None return indetify + suffix
def fetch_poc(self, ssvid): if self.check_account(): try: if ssvid and ssvid.startswith('ssvid-'): ssvid = ssvid.split('ssvid-')[-1] resp = requests.get( 'https://www.seebug.org/api/user/poc_detail?id=%s' % ssvid, headers=self.headers) if resp and resp.status_code == 200 and "code" in resp.json(): poc = resp.json()['code'] return poc except Exception as ex: logger.error(str(ex)) else: return None
def token_is_available(self): if self.token: # distinguish Jwt Token & API Token self.headers['Authorization'] = self.token if len( self.token) < 48 else f'JWT {self.token}' try: resp = requests.get('http://api.ceye.io/v1/identify', headers=self.headers) if resp and resp.status_code == 200 and "identify" in resp.text: return True else: logger.info(resp.text) except Exception as ex: logger.error(str(ex)) return False
def token_is_available(self): if self.token: try: headers = { "X-QuakeToken": self.token, "Content-Type": "application/json" } resp = requests.get('https://quake.360.cn/api/v3/user/info', headers=headers) if resp and resp.status_code == 200 and resp.json( )['code'] == 0: return True except Exception as ex: logger.error(str(ex)) return False
def token_is_available(self): if self.token: try: self.headers['X-QuakeToken'] = self.token resp = requests.get('https://quake.360.cn/api/v3/user/info', headers=self.headers) if 'month_remaining_credit' not in resp.text: logger.info(resp.text) if resp and resp.status_code == 200 and resp.json( )['code'] == 0: return True except Exception as ex: logger.error(str(ex)) return False
def get_redirect_url(url): # TODO: # regex need more test cases meta_regex = '(?is)\<meta[^<>]*?url\s*=([\d\w://\\\\.?=&;%-]*)[^<>]*' body_regex = '''(?is)\<body[^<>]*?location[\s\.\w]*=['"]?([\d\w://\\\\.?=&;%-]*)['"]?[^<>]*''' js_regex = '''(?is)<script.*?>[^<>]*?location\.(?:replace|href|assign)[=\("']*([\d\w://\\\\.?=&;%-]*)[^<>]*?</script>''' resp = requests.get(url) true_url = resp.url for regex in [meta_regex, body_regex, js_regex]: result = re.search(regex, resp.text) if result: redirect_url = result.group(1) true_url = urljoin(url, redirect_url) break return true_url
def search(self, dork, pages=1, resource='host'): search_result = set() try: for page in range(1, pages + 1): url = "https://api.shodan.io/shodan/{0}/search?key={1}&query={2}&page={3}".format(resource, self.token, urllib.parse.quote(dork), page) resp = requests.get(url) if resp and resp.status_code == 200 and "total" in resp.json(): content = resp.json() for match in content['matches']: ans = match['ip_str'] if 'port' in match: ans += ':' + str(match['port']) search_result.add(ans) except Exception as ex: logger.error(str(ex)) return search_result
def search(self, dork, pages=1, resource='web'): search_result = set() try: for page in range(1, pages + 1): url = "https://api.zoomeye.org/{}/search?query={}&page={}&facet=app,os".format( resource, urllib.parse.quote(dork), page) resp = requests.get(url, headers=self.headers) if resp and resp.status_code == 200 and "matches" in resp.json( ): content = resp.json() if resource == 'web': search_result.update( [match['site'] for match in content['matches']]) else: for match in content['matches']: ans = match['ip'] if 'portinfo' in match: ans += ':' + str(match['portinfo']['port']) search_result.add(ans) except Exception as ex: logger.error(str(ex)) return search_result