def requestGithub(keywordsFile, args):
keywordSearches = []
tokenMap = tokens.initTokensMap()
with open(keywordsFile, 'r') as myfile:
for keyword in myfile:
for token in config.GITHUB_TOKENS:
print(colored('[+] Github query : '+config.GITHUB_API_URL + githubQuery +' '+keyword.strip() +config.GITHUB_SEARCH_PARAMS,'yellow'))
# TODO Centralize header management for token auth here (rate-limit more agressive on Github otherwise)
headers = {
'Accept': 'application/vnd.github.v3.text-match+json',
'Authorization': 'token ' + token
}
try:
time.sleep(4);response = requests.get(config.GITHUB_API_URL + githubQuery +' '+keyword.strip() +config.GITHUB_SEARCH_PARAMS, headers=headers)
print('[i] Status code : ' + str(response.status_code))
if response.status_code == 200:
content = parseResults(response.text)
if content:
for rawGitUrl in content.keys():
tokensResult = checkToken(content[rawGitUrl].text, tokenMap)
for token in tokensResult.keys():
displayMessage = displayResults(token, tokensResult, rawGitUrl)
if args.slack:
notifySlack(displayMessage)
if args.wordlist:
writeToWordlist(rawGitUrl, args.wordlist)
break
except UnicodeEncodeError as e:
# TODO improve exception management
print(e.msg)
pass
return keywordSearches
def searchGithub(keywordsFile, args):
tokenMap, tokenCombos = tokens.initTokensMap()
t_keywords = open(keywordsFile).read().split("\n")
pool = Pool( int(args.max_threads) )
pool.map( partial(doSearchGithub,args,tokenMap, tokenCombos), t_keywords )
pool.close()
pool.join()
def searchFilesystem(args):
tokenMap, tokenCombos = tokens.initTokensMap()
log(args.verbose, "[+] scanning", args.mask)
pool = Pool(int(args.max_threads))
pool.map(partial(doSearchFilesystem, args, tokenMap, tokenCombos), findFiles(args.mask))
pool.close()
pool.join()
def searchGithub(keywordsFile, args):
keywordSearches = []
tokenMap = tokens.initTokensMap()
with open(keywordsFile, 'r') as myfile:
for keyword in myfile:
url = config.GITHUB_API_URL + githubQuery + ' ' + keyword.strip(
) + config.GITHUB_SEARCH_PARAMS
response = doRequestGitHub(url, True)
content = parseResults(response.text)
if content:
for rawGitUrl in content.keys():
tokensResult = checkToken(content[rawGitUrl][0].text,
tokenMap)
for token in tokensResult.keys():
displayMessage = displayResults(
token, tokensResult, rawGitUrl, content[rawGitUrl])
if args.slack:
notifySlack(displayMessage)
if args.wordlist:
writeToWordlist(rawGitUrl, args.wordlist)
return keywordSearches
action='store',
dest='wordlist',
help=
'Create a wordlist that fills dynamically with discovered filenames on GitHub'
)
args = parser.parse_args()
if not args.keywordsFile:
print('No keyword (-k or --keyword) file is specified')
exit()
if not args.query:
print('No query (-q or --query) is specified, default query will be used')
args.query = ' '
githubQuery = args.query
keywordsFile = args.keywordsFile
githubQuery = args.query
tokenMap = tokens.initTokensMap()
tokensResult = []
# If wordlist, check if file is binary initialized for mmap
if (args.wordlist):
initFile(args.wordlist)
# Init URL file
initFile(config.GITHUB_URL_FILE)
# Send requests to Github API
responses = requestGithub(keywordsFile, args)
