def requestGithub(keywordsFile, args): keywordSearches = [] tokenMap = tokens.initTokensMap() with open(keywordsFile, 'r') as myfile: for keyword in myfile: for token in config.GITHUB_TOKENS: print(colored('[+] Github query : '+config.GITHUB_API_URL + githubQuery +' '+keyword.strip() +config.GITHUB_SEARCH_PARAMS,'yellow')) # TODO Centralize header management for token auth here (rate-limit more agressive on Github otherwise) headers = { 'Accept': 'application/vnd.github.v3.text-match+json', 'Authorization': 'token ' + token } try: time.sleep(4);response = requests.get(config.GITHUB_API_URL + githubQuery +' '+keyword.strip() +config.GITHUB_SEARCH_PARAMS, headers=headers) print('[i] Status code : ' + str(response.status_code)) if response.status_code == 200: content = parseResults(response.text) if content: for rawGitUrl in content.keys(): tokensResult = checkToken(content[rawGitUrl].text, tokenMap) for token in tokensResult.keys(): displayMessage = displayResults(token, tokensResult, rawGitUrl) if args.slack: notifySlack(displayMessage) if args.wordlist: writeToWordlist(rawGitUrl, args.wordlist) break except UnicodeEncodeError as e: # TODO improve exception management print(e.msg) pass return keywordSearches
def searchGithub(keywordsFile, args): tokenMap, tokenCombos = tokens.initTokensMap() t_keywords = open(keywordsFile).read().split("\n") pool = Pool( int(args.max_threads) ) pool.map( partial(doSearchGithub,args,tokenMap, tokenCombos), t_keywords ) pool.close() pool.join()
def searchFilesystem(args): tokenMap, tokenCombos = tokens.initTokensMap() log(args.verbose, "[+] scanning", args.mask) pool = Pool(int(args.max_threads)) pool.map(partial(doSearchFilesystem, args, tokenMap, tokenCombos), findFiles(args.mask)) pool.close() pool.join()
def searchGithub(keywordsFile, args): keywordSearches = [] tokenMap = tokens.initTokensMap() with open(keywordsFile, 'r') as myfile: for keyword in myfile: url = config.GITHUB_API_URL + githubQuery + ' ' + keyword.strip( ) + config.GITHUB_SEARCH_PARAMS response = doRequestGitHub(url, True) content = parseResults(response.text) if content: for rawGitUrl in content.keys(): tokensResult = checkToken(content[rawGitUrl][0].text, tokenMap) for token in tokensResult.keys(): displayMessage = displayResults( token, tokensResult, rawGitUrl, content[rawGitUrl]) if args.slack: notifySlack(displayMessage) if args.wordlist: writeToWordlist(rawGitUrl, args.wordlist) return keywordSearches
action='store', dest='wordlist', help= 'Create a wordlist that fills dynamically with discovered filenames on GitHub' ) args = parser.parse_args() if not args.keywordsFile: print('No keyword (-k or --keyword) file is specified') exit() if not args.query: print('No query (-q or --query) is specified, default query will be used') args.query = ' ' githubQuery = args.query keywordsFile = args.keywordsFile githubQuery = args.query tokenMap = tokens.initTokensMap() tokensResult = [] # If wordlist, check if file is binary initialized for mmap if (args.wordlist): initFile(args.wordlist) # Init URL file initFile(config.GITHUB_URL_FILE) # Send requests to Github API responses = requestGithub(keywordsFile, args)