def optimize(self, query): url = 'https://www.shodan.io/search/_summary?query={0}'.format(query) utils.print_good("Analyze first page for more result") r = sender.send_get(self.options, url, self.cookies) if r.status_code == 200: soup = utils.soup(r.text) else: return False query_by_cities = [] # check if query have country filter or not if 'country' in query: links = soup.find_all("a") country = utils.get_country_code(utils.url_decode(query)) for link in links: if 'city' in link.get('href'): item = { 'url': link.get('href'), 'city': link.text, 'country': country } utils.print_debug(self.options, item) query_by_cities.append(item) else: links = soup.find_all("a") countries = [] for link in links: if 'country' in link.get('href'): countries.append(utils.get_country_code(utils.url_decode(link.get('href')))) utils.print_debug(self.options, countries) for country in countries: # seding request again to get city country_query = utils.url_encode(' country:"{0}"'.format(country)) url = 'https://www.shodan.io/search/_summary?query={0}{1}'.format( query, country_query) r1 = sender.send_get(self.options, url, self.cookies) utils.random_sleep(5, 8) utils.print_info( "Sleep for couple seconds because Shodan server is really strict") if r1.status_code == 200: soup1 = utils.soup(r1.text) links = soup1.find_all("a") # countries = [] for link in links: if 'city' in link.get('href'): # countries.append(utils.get_city_name( # utils.url_decode(link.get('href')))) item = { 'url': link.get('href'), 'city': link.text, 'country': country } utils.print_debug(self.options, item) query_by_cities.append(item) utils.print_debug(self.options, query_by_cities) return query_by_cities
def sending(self, url): # sending request and return the response utils.print_debug(self.options, url) # catch error when session timeout try: r = sender.send_get(self.options, url, self.cookies) except: r = False if r: response = r.text if self.options['store_content']: ts = str(int(time.time())) raw_file = self.options['raw'] + \ "/fofa/{0}_{1}".format(utils.url_encode( url.replace(self.base_url, '')).replace('/', '_'), ts) utils.just_write(raw_file, response) soup = utils.soup(response) self.analyze(soup) # checking if there is many pages or not page_num = self.check_pages(soup) # if you're log in and have many results if page_num and self.logged_in and not self.options[ 'disable_pages']: utils.print_info("Continue grab more pages") self.pages(page_num)
def brute_country_code(self, query): utils.print_info("Brute the query with country code") if 'country' in query: raw_query = query.replace(utils.get_country_code(utils.url_decode(query)), '[replace]') else: raw_query += utils.url_encode(' country:"[replace]"') for country_code in utils.full_country_code: query = raw_query.replace('[replace]', country_code) url = 'https://www.shodan.io/search?query={1}&page={0}'.format( str(1), query) self.sending(url)
def brute_country_code(self, query): utils.print_info("Brute the query with country code") # clean the country filter raw_query = utils.url_decode(query) if 'location.country' in raw_query: country = utils.get_country_code(raw_query, source='censys') query = raw_query.replace(country, '').replace( 'AND ' + country, '').replace('and ' + country, '') raw_query += ' and location.country_code:"[replace]"' for country_code in utils.full_country_code: query = utils.url_encode( raw_query.replace('[replace]', country_code)) url = 'https://censys.io/ipv4/_search?q={1}&page={0}'.format(str(1), query) self.sending(url)
def tweet(self, tag): results = [] query = '#{0} #{1}'.format(self.query, tag) # @TODO improve by increase the the position url = 'https://twitter.com/search?vertical=default&q={0}&src=unkn'.format( utils.url_encode(query)) r = sender.send_get(self.options, url, cookies=None) if r.status_code == 200: response = r.text # store raw json raw_file_path = self.options['raw'] + '/tweets_{1}_{0}.html'.format( self.query.replace(' ', '_'), tag) if self.options.get('store_content'): utils.just_write(raw_file_path, response) utils.print_debug( self.options, "Writing raw response to: {0}".format(raw_file_path)) soup = utils.soup(response) # Custom here divs = soup.find_all('div', 'original-tweet') for div in divs: content = div.findChildren('p', 'TweetTextSize')[0].text.strip() links = [ x.get('data-expanded-url') for x in div.findChildren('a') if 't.co' in x.get('href') ] # print(links) if len(links) == 0: external_url = 'N/A' else: external_url = '|'.join([str(x) for x in links]) item = { 'Query': self.query, 'Title': query, 'Content': content, 'External_url': external_url, 'Source': url, 'Warning': 'Tweet', 'Raw': raw_file_path } utils.print_debug(self.options, item) results.append(item) return results
def optimize(self, json_response): analytics = json_response.get('aggs') if not analytics: return False # get analytics respoonse url = 'https://www.zoomeye.org/aggs/{0}'.format(analytics) r = sender.send_get(self.options, url, headers=self.headers) if r.status_code == 200: analytics_json = utils.get_json(r.text) else: return False analytics_countries = analytics_json.get('country') raw_query = self.options['zoomeye_query'] clean_query = self.options['zoomeye_query'] if 'country' in raw_query: country_code = utils.get_country_code(utils.url_decode(raw_query)) # clean country and subdivisions if it exist clean_query = raw_query.replace(' +country:', '').replace( '"{0}"'.format(str(country_code)), '') for country_item in analytics_countries: utils.print_info( "Optimize query by filter with coutry: {0}".format( country_item.get('name'))) # loop through city for city in country_item.get('subdivisions'): if 'country' in raw_query: real_query = raw_query + ' +subdivisions:"{0}"'.format( city.get('name')) else: real_query = clean_query + \ ' +country:"{0}"'.format(country_item.get('name')) + \ ' +subdivisions:"{0}"'.format(city.get('name')) query = utils.url_encode(real_query) url = 'https://www.zoomeye.org/search?q={0}&t=host'.format( query) r = sender.send_get(self.options, url, headers=self.headers) if r and r.status_code == 200: json_response = utils.get_json(r.text) self.analyze(json_response)
def brute_country_code(self, query): utils.print_info("Brute the query with country code") # && country=US raw_query = utils.just_b64_decode(utils.url_decode(query)) if 'country' in query: raw_query = query.replace( utils.get_country_code(raw_query, source='fofa'), '[replace]') else: raw_query += '&& country:"[replace]"' for country_code in utils.full_country_code: query = raw_query.replace('[replace]', country_code) query = utils.url_encode(utils.just_b64_encode(query)) url = 'https://fofa.so/result?page={0}&qbase64={1}'.format( str(1), query) self.sending(url)
def initial(self): # prepare url query = utils.url_encode(self.options['shodan_query']) url = 'https://www.shodan.io/search?query={1}&page={0}'.format(str(1), query) self.sending(url) # brute the country if self.options['brute']: self.brute_country_code(query) # repeat the routine with filter by city query_by_cities = self.optimize(query) if query_by_cities: for item in query_by_cities: utils.print_info( "Get more result by filter with {0} city".format(item.get('city'))) self.sending(item.get('url'))
def pages(self, page_num): for i in range(2, int(page_num) + 1): utils.print_info("Get more result from page: {0}".format(str(i))) utils.random_sleep(1, 2) query = utils.url_encode(self.options['censys_query']) url = 'https://censys.io/ipv4/_search?q={1}&page={0}'.format( str(i), query) r = sender.send_get(self.options, url, self.cookies) if r.status_code == 200: response = r.text if 'class="alert alert-danger"' in response: utils.print_bad("Reach to the limit at page {0}".format( str(i))) return else: soup = utils.soup(response) self.analyze(soup)
def sending(self, url): # sending request and return the response r = sender.send_get(self.options, url, self.cookies) if r: response = r.text if self.options['store_content']: ts = str(int(time.time())) raw_file = self.options['raw'] + \ "/shodan/{0}_{1}".format(utils.url_encode( url.replace(self.base_url, '')).replace('/', '_'), ts) utils.just_write(raw_file, response) soup = utils.soup(response) self.analyze(soup) # checking if there is many pages or not if self.logged_in and not self.options['disable_pages']: utils.print_info("Continue grab more pages") self.pages(self.get_num_pages(url))
def pages(self, page_num): for i in range(2, int(page_num) + 1): utils.random_sleep(1, 2) utils.print_info("Get more result from page: {0}".format(str(i))) query = utils.url_encode(self.options['zoomeye_query']) url = 'https://www.zoomeye.org/search?q={0}&t=host&p={1}'.format( query, str(i)) r = sender.send_get(self.options, url, headers=self.headers) if r.status_code == 200: response = r.text if '"msg": "forbidden"' in response: utils.print_bad("Reach to the limit at page {0}".format( str(i))) return else: json_response = utils.get_json(response) self.analyze(json_response) self.optimize(json_response)
def pages(self, page_num): for i in range(2, int(page_num) + 1): utils.print_info("Get more result from page: {0}".format(str(i))) query = utils.url_encode( utils.just_b64_encode(self.options['fofa_query'])) url = 'https://fofa.so/result?page={0}&qbase64={1}'.format( str(i), query) utils.print_debug(self.options, url) r = sender.send_get(self.options, url, self.cookies) if r.status_code == 200: response = r.text if 'class="error"' in response: utils.print_bad("Reach to the limit at page {0}".format( str(i))) return else: soup = utils.soup(response) self.analyze(soup)
def pages(self, page_num): for i in range(2, int(page_num) + 1): utils.print_info("Sleep for couple seconds because Shodan server is really strict") utils.random_sleep(3, 6) utils.print_info("Get more result from page: {0}".format(str(i))) query = utils.url_encode(self.options['shodan_query']) url = 'https://www.shodan.io/search?query={1}&page={0}'.format( str(i), query) r = sender.send_get(self.options, url, self.cookies) if r.status_code == 200: response = r.text if 'class="alert alert-error text-center"' in response: utils.print_bad( "Reach to the limit at page {0}".format(str(i))) return else: soup = utils.soup(response) self.analyze(soup)
def initial(self): product = utils.url_encode(self.query) url = 'https://www.cvedetails.com/product-search.php?vendor_id=0&search={0}'.format( product) # get summary table products = [] r = sender.send_get(self.options, url, cookies=None) if r.status_code == 200: response = r.text if 'class="errormsg"' in response: utils.print_bad("No entry found for: {0}".format(self.query)) return summary_table = utils.soup(response).find_all("table", "listtable") # <table class = "listtable" if summary_table: trs = summary_table[0].findChildren('tr') if len(trs) <= 1: utils.print_bad( "No entry found for: {0}".format(self.query)) return for tr in trs[1:]: for td in tr.findChildren('td'): if td.a: if 'See all vulnerabilities' in td.a.get('title'): products.append(td.a.get('href')) final = [] # if found product and have vulnerabilities, go get it if products: for url in products: results = self.sending(self.baseURL + url) if results: final.extend(results) # self.details(products) # print(final) # write final output self.conclude(final)
def initial(self): # prepare url query = utils.url_encode( utils.just_b64_encode(self.options['fofa_query'])) url = 'https://fofa.so/result?page={0}&qbase64={1}'.format( str(1), query) self.sending(url) # brute the country if self.options['brute']: self.brute_country_code(query) # repeat the routine with filter by city query_by_cities = self.optimize(query) if query_by_cities: for item in query_by_cities: utils.print_info( "Get more result by filter with {0} city".format( item.get('city'))) self.sending(item.get('url')) time.sleep(1)
def sending(self, url): # sending request and return the response r1 = sender.send_get(self.options, url, headers=self.headers) if r1: response = r1.text if self.options['store_content']: ts = str(int(time.time())) raw_file = self.options['raw'] + \ "/zoomeye/{0}_{1}".format(utils.url_encode( url.replace(self.base_url, '')).replace('/', '_'), ts) utils.just_write(raw_file, response) json_response = utils.get_json(response) self.analyze(json_response) # loop throuh pages if you're logged in page_num = self.get_num_pages(json_response) if self.logged_in and page_num and int(page_num) > 1: self.pages(page_num) # get aggs and found more result self.optimize(json_response)
def sending(self, url): # sending request and return the response r = sender.send_get(self.options, url, self.cookies) if r: response = r.text if 'ratelimit' in response: utils.print_bad('Looks like you get block from Censys. Consider using Proxy') return False if self.options['store_content']: ts = str(int(time.time())) raw_file = self.options['raw'] + \ "/censys/{0}_{1}".format(utils.url_encode( url.replace(self.base_url, '')).replace('/', '_'), ts) utils.just_write(raw_file, response) soup = utils.soup(response) self.analyze(soup) # checking if there is many pages or not if not self.options['disable_pages']: utils.print_info("Continue grab more pages") self.pages(self.get_num_pages(soup))
def parsing_config(config_path, args): options = {} # some default path github_api_key = str(os.getenv("GITROB_ACCESS_TOKEN")) cwd = str(os.getcwd()) # just hardcode if gopath not loaded go_path = cwd + "/plugins/go" # go_path = str(os.getenv("GOPATH")) + "/bin" # if "None" in go_path: # go_path = cwd + "/plugins/go" if args.slack: bot_token = str(os.getenv("SLACK_BOT_TOKEN")) else: bot_token = None log_channel = str(os.getenv("LOG_CHANNEL")) status_channel = str(os.getenv("STATUS_CHANNEL")) report_channel = str(os.getenv("REPORT_CHANNEL")) stds_channel = str(os.getenv("STDS_CHANNEL")) verbose_report_channel = str(os.getenv("VERBOSE_REPORT_CHANNEL")) if os.path.isfile(config_path): utils.print_info('Config file detected: {0}'.format(config_path)) # config to logging some output config = ConfigParser(interpolation=ExtendedInterpolation()) config.read(config_path) else: utils.print_info('New config file created: {0}'.format(config_path)) shutil.copyfile(cwd + '/template-config.conf', config_path) config = ConfigParser(interpolation=ExtendedInterpolation()) config.read(config_path) if args.workspace: workspaces = os.path.abspath(args.workspace) else: workspaces = cwd + "/workspaces/" config.set('Enviroments', 'cwd', cwd) config.set('Enviroments', 'go_path', go_path) config.set('Enviroments', 'github_api_key', github_api_key) config.set('Enviroments', 'workspaces', str(workspaces)) if args.debug: config.set('Slack', 'bot_token', 'bot_token') config.set('Slack', 'log_channel', 'log_channel') config.set('Slack', 'status_channel', 'status_channel') config.set('Slack', 'report_channel', 'report_channel') config.set('Slack', 'stds_channel', 'stds_channel') config.set('Slack', 'verbose_report_channel', 'verbose_report_channel') else: config.set('Slack', 'bot_token', str(bot_token)) config.set('Slack', 'log_channel', log_channel) config.set('Slack', 'status_channel', status_channel) config.set('Slack', 'report_channel', report_channel) config.set('Slack', 'stds_channel', stds_channel) config.set('Slack', 'verbose_report_channel', verbose_report_channel) # Mode config of the tool if args.slow and args.slow == 'all': speed = "slow" else: speed = "quick" module = str(args.module) debug = str(args.debug) force = str(args.force) config.set('Mode', 'speed', speed) config.set('Mode', 'module', module) config.set('Mode', 'debug', debug) config.set('Mode', 'force', force) # Target stuff # parsing agument git_target = args.git if args.git else None burpstate_target = args.burp if args.burp else None target_list = args.targetlist if args.targetlist else None company = args.company if args.company else None output = args.output if args.output else None target = args.target if args.target else None strip_target = target if target else None ip = target if target else None workspace = target if target else None # get direct input as single or a file direct_input = args.input if args.input else None direct_input_list = args.inputlist if args.inputlist else None # target config if args.target: target = args.target # set target is direct input if not specific elif direct_input or direct_input_list: if direct_input: # direct_target = utils.url_encode(direct_input) direct_target = direct_input if direct_input_list: direct_target = os.path.basename(direct_input_list) target = direct_target output = args.output if args.output else utils.strip_slash( os.path.splitext(target)[0]) company = args.company if args.company else utils.strip_slash( os.path.splitext(target)[0]) else: target = None # parsing some stuff related to target if target: # get the main domain of the target strip_target = utils.get_domain(target) if '/' in strip_target: strip_target = utils.strip_slash(strip_target) output = args.output if args.output else strip_target company = args.company if args.company else strip_target # url encode to make sure it can be send through API workspace = workspaces + strip_target workspace = utils.url_encode(workspace) # check connection to target if not direct_input and not direct_input_list: try: ip = socket.gethostbyname(strip_target) except: ip = None utils.print_bad( "Something wrong to connect to {0}".format(target)) else: ip = None try: # getting proxy from args proxy = args.proxy if args.proxy else None proxy_file = args.proxy_file if args.proxy_file else None config.set('Proxy', 'proxy', str(proxy)) config.set('Proxy', 'proxy_file', str(proxy_file)) if config['Proxy']['proxy_cmd'] == 'None': # only works for Kali proxychains, change it if you on other OS proxy_cmd = "proxychains -f {0}".format(proxy_file) config.set('Proxy', 'proxy_cmd', str(proxy_cmd)) except: utils.print_info( "Your config file seem to be outdated, Backup it and delete it to regenerate the new one" ) config.set('Target', 'input', str(direct_input)) config.set('Target', 'input_list', str(direct_input_list)) config.set('Target', 'git_target', str(git_target)) config.set('Target', 'burpstate_target', str(burpstate_target)) config.set('Target', 'target_list', str(target_list)) config.set('Target', 'output', str(output)) config.set('Target', 'target', str(target)) config.set('Target', 'strip_target', str(strip_target)) config.set('Target', 'company', str(company)) config.set('Target', 'ip', str(ip)) config.set('Enviroments', 'workspace', str(workspace)) # create workspace folder for the target utils.make_directory(workspace) # set the remote API if args.remote: remote_api = args.remote config.set('Server', 'remote_api', remote_api) # set credentials as you define from agurments if args.auth: # user:pass creds = args.auth.strip().split(":") username = creds[0] password = creds[1] config.set('Server', 'username', username) config.set('Server', 'password', password) else: # set random password if default password detect if config['Server']['password'] == 'super_secret': new_pass = hashlib.md5(str(int( time.time())).encode()).hexdigest()[:6] config.set('Server', 'password', new_pass) # save the config with open(config_path, 'w') as configfile: config.write(configfile) config = ConfigParser(interpolation=ExtendedInterpolation()) config.read(config_path) sections = config.sections() options['CONFIG_PATH'] = os.path.abspath(config_path) for sec in sections: for key in config[sec]: options[key.upper()] = config.get(sec, key) # if args.slow and args.slow != 'all': options['SLOW'] = args.slow # parsing proxy stuff if options.get('PROXY') or options.get('PROXY_FILE'): proxy_parsing(options) else: # just for the old config options['PROXY'] = "None" options['PROXY_FILE'] = "None" options = clean_up(options) return options
def initial(self): # prepare url query = utils.url_encode(self.options['zoomeye_query']) url = 'https://www.zoomeye.org/search?q={0}&t=host&p=1'.format(query) self.sending(url)