def scrape(scrape_input_method, displaymode, limiter, repo_crawl, verbosity):
if scrape_input_method.lower() == 'm':
url = input("Enter the URL: ")
urlpage = connect(url)
if urlpage == 'connection failed':
lib.PrintError(
"Connection to specified URL could not be established.")
exit()
else:
lib.PrintStatus('Status: [200], Searching for API Keys...')
if repo_crawl is False:
search_execute(displaymode, urlpage)
else:
repository_list = get_repos(url)
file_addresses = traverse_repos(repository_list, verbosity)
executor = ThreadPoolExecutor(max_workers=len(file_addresses))
for addr in set(file_addresses):
urlpage = connect(addr)
executor.submit(search_execute(displaymode, urlpage))
sleep(limiter)
lib.PrintSuccess("Scanning complete.")
else:
while True:
url_file = input("Enter the full path to the input file: ")
if isfile(url_file) is True:
break
elif str(url_file) == "":
lib.DoNothing()
else:
lib.PrintError("No Such File Found.")
continue
with open(url_file) as ufile:
count = 0
for line in ufile.readlines():
if repo_crawl is False:
count += 1
urlpage = connect(line.rstrip())
if urlpage == 'connection failed':
lib.PrintFailure(
f"[Line: {count}] Connection failed on host {line}"
)
else:
search_execute(displaymode, urlpage)
sleep(limiter)
else:
repository_list = get_repos(line)
file_addresses = traverse_repos(repository_list, verbosity)
executor = ThreadPoolExecutor(
max_workers=len(file_addresses))
for addr in set(file_addresses):
urlpage = connect(addr)
executor.submit(search_execute(displaymode, urlpage))
sleep(limiter)
def main():
lib.PrintTitle()
while True:
conf = input("[L]oad configuration file or do [m]anual setup: ")
if conf not in ['l', 'm']:
lib.PrintError("Invalid Input")
continue
elif conf.lower() == 'l':
vars_dict = load_config()
if vars_dict['search_type'] == 's':
shodan_scan(vars_dict['shodan_key'], vars_dict['local_dir'])
lib.PrintSuccess("Scan Complete")
break
elif vars_dict['search_type'] == 'f':
file_scan(vars_dict['address_file'], vars_dict['local_dir'])
lib.PrintSuccess("Scan Complete")
break
else:
masscan_scan(vars_dict['local_dir'])
lib.PrintSuccess("Scan Complete")
break
else:
vars_dict = manual_setup()
if vars_dict['search_type'] == 's':
shodan_scan(vars_dict['shodan_key'], vars_dict['local_dir'])
lib.PrintSuccess("Scan Complete")
break
elif vars_dict['search_type'] == 'f':
file_scan(vars_dict['address_file'], vars_dict['local_dir'])
lib.PrintSuccess("Scan Complete")
break
else:
masscan_scan(vars_dict['local_dir'])
lib.PrintSuccess("Scan Complete")
break
def redis_search(displaymode, page, repo_crawl, verbosity):
if repo_crawl is False:
lib.PrintStatus("Scanning for Redis secrets...")
elif repo_crawl is True and verbosity == 'on':
lib.PrintStatus("Scanning for Redis secrets...")
pagetext = page.text
redis_pattern = r'redis://[0-9a-zA-Z:@.\\-]+'
redis_artifacts = ['REDIS_PASSWORD', 'REDIS_CACHE_DATABASE', 'REDIS_HOST', 'REDIS_DATABASE']
for k in re.findall(redis_pattern, pagetext):
if displaymode == 's' or 'b':
lib.PrintHighSeverity('Warning: High Severity Item Found')
redis_output = f'{curdir}/Output/Redis/RedisLinks.txt'
if not exists(dirname(redis_output)):
try:
makedirs(dirname(redis_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(redis_output, 'a') as gofile:
gofile.write(f'Potential link: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential link: {k}')
lib.PrintHighSeverity('Warning: High Severity Item Found')
for ra in set(redis_artifacts):
if ra in pagetext:
lib.PrintHighSeverity('Warning: High Severity Item Found')
if displaymode == 's' or 'b':
redis_artifacts_output = f'{curdir}/Output/Redis/RedisArtifacts.txt'
if not exists(dirname(redis_artifacts_output)):
try:
makedirs(dirname(redis_artifacts_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(redis_artifacts_output, 'a') as rafile:
rafile.write(f'Artifact found: {ra}')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Artifact Found: {ra}')
def ftp_operations(host, output_location): # TODO: Find out how to check and record file sizes in relation to FTP
if output_location.endswith('/') is False:
output_location = output_location + '/'
ftp_connection = ftplib.FTP(host)
try:
ftp_connection.login()
lib.PrintSuccess('Login Status: 200')
lib.PrintStatus(f'Exfiltrating files to {output_location}')
filenames = ftp_connection.nlst()
for filename in filenames:
local_filename = path.join(output_location, filename)
file = open(local_filename, 'wb')
ftp_connection.retrbinary('RETR ' + filename, file.write)
except Exception as e:
lib.PrintError(f'{e}')
def slack_api_search(displaymode, page):
lib.PrintStatus("Scanning for slack API keys...")
pagetext = page.text
slack_api_pattern = r"xoxp-\\d+-\\d+-\\d+-[0-9a-f]+"
for k in re.findall(slack_api_pattern, pagetext):
if displaymode == 's' or 'b':
sapi_output = f'{curdir}\\Output\\SlackAPIPotentialKeys.txt'
if not exists(dirname(sapi_output)):
try:
makedirs(dirname(sapi_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(sapi_output, 'a') as gofile:
gofile.write(f'Potential Key: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Key: {k}')
def slack_bot_search(displaymode, page):
lib.PrintStatus("Scanning for slack bot tokens...")
pagetext = page.text
slack_bot_pattern = r"xoxb-\\d+-[0-9a-zA-Z]+"
for k in re.findall(slack_bot_pattern, pagetext):
if displaymode == 's' or 'b':
slack_bot_output = f'{curdir}/Output/SlackBotPotentialTokens.txt'
if not exists(dirname(slack_bot_output)):
try:
makedirs(dirname(slack_bot_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(slack_bot_output, 'a') as gofile:
gofile.write(f'Potential Token: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Token: {k}')
def nonspecific_api_search(displaymode, page):
lib.PrintStatus("Scanning for nonspecific API keys...")
pagetext = page.text
nonspecific_pattern = r"[a|A][p|P][i|I][_]?[k|K][e|E][y|Y].{0,30}['\"\\s][0-9a-zA-Z]{32,45}['\"\\s]"
for k in re.findall(nonspecific_pattern, pagetext):
if displaymode == 's' or 'b':
nonspecific_output = f'{curdir}/Output/NonspecificPotentialKeys.txt'
if not exists(dirname(nonspecific_output)):
try:
makedirs(dirname(nonspecific_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(nonspecific_output, 'a') as gofile:
gofile.write(f'Potential Key: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Key: {k}')
def twilio_search(displaymode, page):
lib.PrintStatus("Scanning for twilio keys...")
pagetext = page.text
twilio_pattern = r"SK[a-z0-9]{32}"
for k in re.findall(twilio_pattern, pagetext):
if displaymode == 's' or 'b':
twilio_output = f'{curdir}/Output/TwilioKeys.txt'
if not exists(dirname(twilio_output)):
try:
makedirs(dirname(twilio_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(twilio_output, 'a') as gofile:
gofile.write(f'Potential Key: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Key: {k}')
def github_search(displaymode, page):
lib.PrintStatus("Searching for Github keys...")
github_api = r"[g|G][i|I][t|T][h|H][u|U][b|B].{0,30}['\"\\s][0-9a-zA-Z]{35,40}['\"\\s]"
pagetext = page.text
for k in re.findall(github_api, pagetext):
if displaymode == 's' or 'b':
github_output = f'{curdir}/Output/GithubPotentialKeys.txt'
if not exists(dirname(github_output)):
try:
makedirs(dirname(github_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(github_output, 'a') as gofile:
gofile.write(f'Potential Key: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Key: {k}')
def heroku_search(displaymode, page):
lib.PrintStatus("Scanning for Heroku API keys...")
pagetext = page.text
heroku_pattern = r"[h|H][e|E][r|R][o|O][k|K][u|U].{0,30}[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}"
for k in re.findall(heroku_pattern, pagetext):
if displaymode == 's' or 'b':
heroku_output = f'{curdir}/Output/HerokuKeys.txt'
if not exists(dirname(heroku_output)):
try:
makedirs(dirname(heroku_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(heroku_output, 'a') as gofile:
gofile.write(f'Potential Key: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Key: {k}')
def discord_webhook_search(displaymode, page):
lib.PrintStatus("Scanning for discord webhooks...")
pagetext = page.text
discord_webhook_pattern = r"(https:\/\/discordapp\.com\/api\/webhooks\/[\d]+\/[\w]+)"
for k in re.findall(discord_webhook_pattern, pagetext):
if displaymode == 's' or 'b':
discord_webhook_output = f'{curdir}/Output/DiscordWebhooks.txt'
if not exists(dirname(discord_webhook_output)):
try:
makedirs(dirname(discord_webhook_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(discord_webhook_output, 'a') as gofile:
gofile.write(f'Potential Hook: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Hook: {k}')
def discord_nitro_search(displaymode, page):
lib.PrintStatus("Scanning for discord nitro links...")
pagetext = page.text
discord_nitro_pattern = r"(https:\/\/discord\.gift\/.+[a-z{1,16}])"
for k in re.findall(discord_nitro_pattern, pagetext):
if displaymode == 's' or 'b':
discord_nitro_output = f'{curdir}/Output/DiscordNitroPotentialLinks.txt'
if not exists(dirname(discord_nitro_output)):
try:
makedirs(dirname(discord_nitro_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(discord_nitro_output, 'a') as gofile:
gofile.write(f'Potential link: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential link: {k}')
def slack_webhook_search(displaymode, page):
lib.PrintStatus("Scanning for slack webhooks...")
pagetext = page.text
slack_webhook_pattern = r"https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}"
for k in re.findall(slack_webhook_pattern, pagetext):
if displaymode == 's' or 'b':
slack_webhook_output = f'{curdir}/Output/SlackWebhooks.txt'
if not exists(dirname(slack_webhook_output)):
try:
makedirs(dirname(slack_webhook_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(slack_webhook_output, 'a') as gofile:
gofile.write(f'Potential Hook: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Hook: {k}')
def discord_bot_search(displaymode, page):
lib.PrintStatus("Scanning for discord bot tokens...")
pagetext = page.text
discord_token_pattern = r"([\w\-\.]+[\-\.][\w\-\.]+)"
for k in re.findall(discord_token_pattern, pagetext):
if displaymode == 's' or 'b':
discord_bot_output = f'{curdir}/Output/DiscordBotPotentialTokens.txt'
if not exists(dirname(discord_bot_output)):
try:
makedirs(dirname(discord_bot_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(discord_bot_output, 'a') as gofile:
gofile.write(f'Potential Token: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Token: {k}')
def google_api_search(displaymode, page):
lib.PrintStatus("Scanning for google API keys...")
pagetext = page.text
google_api_pattern = r'AIzaSy[0-9a-zA-Z_\\-]{33}'
for k in re.findall(google_api_pattern, pagetext):
if displaymode == 's' or 'b':
gapi_output = f'{curdir}/Output/GoogleAPIPotentialKeys.txt'
if not exists(dirname(gapi_output)):
try:
makedirs(dirname(gapi_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(gapi_output, 'a') as gofile:
gofile.write(f'Potential Key: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Key: {k}')
def AWS_search(displaymode, page):
lib.PrintStatus("Searching for AWS Access Keys...")
aws_pattern = r"AKIA[0-9A-Z]{16}"
pagetext = page.text
for k in re.findall(aws_pattern, pagetext):
if displaymode == 's' or 'b':
aws_output = f'{curdir}/Output/AWSPotentialTokens.txt'
if not exists(dirname(aws_output)):
try:
makedirs(dirname(aws_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(aws_output, 'a') as gofile:
gofile.write(f'Potential Tokens: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Token: {k}')
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def redis_search(displaymode, page):
lib.PrintStatus("Scanning for Redis URLs...")
pagetext = page.text
redis_pattern = r'redis://[0-9a-zA-Z:@.\\-]+'
for k in re.findall(redis_pattern, pagetext):
if displaymode == 's' or 'b':
redis_output = f'{curdir}/Output/RedisLinks.txt'
if not exists(dirname(redis_output)):
try:
makedirs(dirname(redis_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(redis_output, 'a') as gofile:
gofile.write(f'Potential link: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential link: {k}')
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def facebook_OAUTH(displaymode, page):
lib.PrintStatus("Scanning for facebook OAUTH secrets...")
pagetext = page.text
fauth_pattern = r"[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].{0,30}['\"\\s][0-9a-f]{32}['\"\\s]"
for k in re.findall(fauth_pattern, pagetext):
if displaymode == 's' or 'b':
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
fauth_output = f'{curdir}/Output/FacebookOAUTHSecrets.txt'
if not exists(dirname(fauth_output)):
try:
makedirs(dirname(fauth_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(fauth_output, 'a') as gofile:
gofile.write(f'Potential Secret: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Secret: {k}')
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def ssh_keys_search(displaymode, page):
lib.PrintStatus("Scanning for SSH Keys...")
pagetext = page.text
ssh_keys_identifiers = ["-----BEGIN OPENSSH PRIVATE KEY-----", "-----BEGIN DSA PRIVATE KEY-----", "-----BEGIN EC PRIVATE KEY-----"]
for pattern in set(ssh_keys_identifiers):
if pattern in pagetext:
if displaymode == 's' or 'b':
ssh_output = f'{curdir}/Output/SSHKeys.txt'
if not exists(dirname(ssh_output)):
try:
makedirs(dirname(ssh_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(ssh_output, 'a') as gofile:
gofile.write(f'SSH Key: {pattern}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'SSH Key: {pattern}')
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def google_oauth_search(displaymode, page):
lib.PrintStatus("Scanning for google OAUTH secrets...")
pagetext = page.text
gauth_pattern = r"(\"client_secret\":\"[a-zA-Z0-9-_]{24}\")"
for k in re.findall(gauth_pattern, pagetext):
if displaymode == 's' or 'b':
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
gauth_output = f'{curdir}/Output/GoogleOAUTHSecrets.txt'
if not exists(dirname(gauth_output)):
try:
makedirs(dirname(gauth_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(gauth_output, 'a') as gofile:
gofile.write(f'Potential Secret: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Secret: {k}')
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def google_access_token_search(displaymode, page):
lib.PrintStatus("Scanning for google access tokens...")
pagetext = page.text
gat_pattern = r'ya29.[0-9a-zA-Z_\\-]{68}'
for k in re.findall(gat_pattern, pagetext):
if displaymode == 's' or 'b':
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
gat_output = f'{curdir}/Output/GoogleAccessPotentialTokens.txt'
if not exists(dirname(gat_output)):
try:
makedirs(dirname(gat_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(gat_output, 'a') as gofile:
gofile.write(f'Potential Token: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Token: {k}')
lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def generic_search(key, displaymode, page, repo_crawl, verbosity):
if repo_crawl is False:
lib.PrintStatus(f"Searching for {key[0]} keys...")
elif repo_crawl is True and verbosity == 'on':
lib.PrintStatus(f"Searching for {key[0]} keys...")
pagetext = page.text
for k in re.findall(lib.patterns_dict[key], pagetext):
if displaymode == 's' or 'b':
if key[1] is True:
lib.PrintHighSeverity("Warning: High Severity Item Found")
gen_output = f'{curdir}/Output/{key[0]}.txt'
if not exists(dirname(gen_output)):
try:
makedirs(dirname(gen_output))
except OSError as racecondition:
if racecondition.errno != errno.EEXIST:
raise
with open(gen_output, 'a') as gofile:
gofile.write(f'Potential Key: {k}\n')
elif displaymode == 'p' or 'b':
lib.PrintSuccess(f'Potential Key: {k}')
keylisting = parser.getboolean('initial_vars', 'keylisting')
key_list = parser.get('initial_vars', 'key_list')
arch_mode = parser.get('initial_vars', 'arch_mode')
ArchiveSearch(stop_input, arch_mode)
else:
lib.PrintError("No such file found")
continue
elif configchoice.lower() == 'n':
while True:
workpath = input(
"Enter the path you wish to save text documents to (enter curdir for current directory): "
)
if workpath.lower() == 'curdir':
workpath = curdir
if path.isdir(workpath):
lib.PrintSuccess("Valid Path...")
if workpath.endswith('\\'):
pass
else:
workpath = workpath + str('\\')
break
else:
lib.PrintError("Invalid path, check input...")
continue
while True:
try:
stopinput_input = input(
"Run in a constant loop? [y]/[n]: ")
if stopinput_input.lower() == 'y':
stop_input = True
def ArchiveSearch(stop, amode):
arch_runs = 0
while True:
if arch_runs > 0:
lib.PrintStatus("Runs: " + str(arch_runs))
if arch_runs >= stop and stop is False:
lib.PrintSuccess("Runs Complete, Operation Finished... [" +
str(datetime.now().strftime('%X')) + "]")
exit()
else:
lib.PrintStatus("Pastes fetched, cooling down for " +
str(cooldown) + " seconds... [" +
str(datetime.now().strftime('%X')) + "]")
sleep(cooldown / 2)
lib.PrintStatus("Halfway through at [" +
str(datetime.now().strftime('%X')) + "]")
sleep(cooldown / 2)
lib.PrintStatus("resuming... [" +
str(datetime.now().strftime('%X')) + "]")
if arch_runs < stop or stop is True:
arch_page, arch_filename = archive_connect()
arch_soup = BeautifulSoup(arch_page.text, 'html.parser')
sleep(2)
lib.PrintStatus("Getting archived pastes... [" +
str(datetime.now().strftime('%X')) + "]")
if AccessDeniedError in arch_page.text:
lib.PrintError(
"IP Temporarily suspending, pausing until the ban is lifted. Estimated time: one hour... ["
+ str(datetime.now().strftime('%X')) + "]")
sleep(cooldown)
lib.PrintStatus("Process resumed... [" +
str(datetime.now().strftime('%X')) + "]")
continue
else:
pass
lib.PrintStatus("Finding params... [" +
str(datetime.now().strftime('%X')) + "]")
table = arch_soup.find(
"table",
class_="maintable") # Fetch the table of recent pastes
while True:
try:
tablehrefs = table.findAll(
'a', href=True) # Find the <a> tags for every paste
break
except AttributeError:
lib.PrintError(
"IP Temporarily suspending, pausing until the ban is lifted. Estimated time: one hour... ["
+ str(datetime.now().strftime('%X')) + "]")
sleep(cooldown)
lib.PrinrError("Process resumed... [" +
str(datetime.now().strftime('%X')) + "]")
continue
for h in tablehrefs:
proch = h['href'] # fetch the URL param for each paste
lib.PrintSuccess("params fetched... [" +
str(datetime.now().strftime('%X')) + "]")
lib.PrintStatus("Acting on param " + str(proch) + "... [" +
str(datetime.now().strftime('%X')) + "]")
full_archpage, full_arch_url = parameter_connect(proch)
sleep(5)
item_soup = BeautifulSoup(full_archpage.text, 'html.parser')
unprocessed = item_soup.find(
'textarea') # Fetch the raw text in the paste.
for tag in taglist:
unprocessed = str(unprocessed).replace(
tag, ""
) # process the raw text by removing all html elements
if amode == 'r':
if path.isdir(workpath) is True:
if blacklisting is True:
flagged = False
compare_text = re.sub(
r'\s+', '', unprocessed
) # strip all whitespace for comparison
for b in blacklist:
b = re.sub(
r'\s+', '',
b) # strip all whitespace for comparison
if b.lower() in compare_text.lower():
lib.PrintStatus(
"Blacklisted phrase detected, passing..."
)
flagged = True
if flagged is True:
continue
else:
arch_final_file = codecs.open(
str(workpath) + str(full_arch_url).replace(
":", "-").replace(":", "-").replace(
"/", "-") + ".txt", 'w+', 'utf-8')
arch_final_file.write(unprocessed)
arch_final_file.close()
arch_runs += 1
continue
elif blacklisting is False:
arch_final_file = codecs.open(
str(workpath) +
str(full_arch_url).replace(":", "-").replace(
":", "-").replace("/", "-") + ".txt", 'w+',
'utf-8')
arch_final_file.write(unprocessed)
arch_final_file.close()
arch_runs += 1
continue
else:
lib.PrintStatus("Making directory... [" +
str(datetime.now().strftime('%X')) +
"]")
if blacklisting is True:
flagged = False
compare_text = re.sub(
r'\s+', '', unprocessed
) # strip all whitespace for comparison
for b in blacklist:
b = re.sub(
r'\s+', '',
b) # strip all whitespace for comparison
if b.lower() in compare_text.lower():
lib.PrintStatus(
"Blacklisted phrase detected, passing..."
)
flagged = True
if flagged is True:
continue
else:
arch_final_file = codecs.open(
str(workpath) + str(full_arch_url).replace(
":", "-").replace(":", "-").replace(
"/", "-") + ".txt", 'w+', 'utf-8')
arch_final_file.write(unprocessed)
arch_final_file.close()
arch_runs += 1
continue
elif blacklisting is False:
arch_final_file = codecs.open(
str(workpath) +
str(full_arch_url).replace(":", "-").replace(
":", "-").replace("/", "-") + ".txt", 'w+',
'utf-8')
arch_final_file.write(unprocessed)
arch_final_file.close()
arch_runs += 1
continue
elif amode == 'f':
if path.isdir(workpath) is True:
lib.PrintStatus("Running engine... [" +
str(datetime.now().strftime('%X')) +
"]")
if blacklisting is True:
flagged = False
compare_text = re.sub(
r'\s+', '', unprocessed
) # strip all whitespace for comparison
for b in blacklist:
b = re.sub(
r'\s+', '',
b) # strip all whitespace for comparison
if b.lower() in compare_text.lower():
lib.PrintStatus(
"Blacklisted phrase detected, passing..."
)
flagged = True
if flagged is True:
continue
else:
archive_engine(unprocessed, keylisting,
reglisting)
arch_runs += 1
continue
else:
lib.PrintStatus(
"Running engine... [" +
str(datetime.now().strftime('%X')) + "]")
archive_engine(unprocessed, keylisting, reglisting)
arch_runs += 1
continue
else:
lib.PrintSuccess("Operation Finished... [" +
str(datetime.now().strftime('%X')) + "]")
break
def scrape(scrape_input_method, displaymode, limiter, repo_crawl, link_type, directory_filtering, blacklisted_directories, verbosity):
if scrape_input_method.lower() == 'm':
url = input("Enter the URL: ")
if url[len(url)-1] == ' ':
url = url[:len(url)-1]
urlpage = connect(url)
if urlpage == 'connection failed':
lib.PrintError("Connection to specified URL could not be established.")
exit()
else:
lib.PrintStatus('Status: [200], Searching for API Keys...')
if repo_crawl is False:
better_search_execute(displaymode, urlpage, repo_crawl, verbosity)
else:
if link_type == 'profile':
resources = get_repos(url)
file_addresses = traverse_repos(resources, link_type, directory_filtering, blacklisted_directories, verbosity)
elif link_type == 'repo':
file_addresses = traverse_repos(url, link_type, directory_filtering, blacklisted_directories, verbosity)
if len(file_addresses) > 0:
executor = ThreadPoolExecutor(max_workers=len(file_addresses))
else:
lib.PrintError("Fatal Error: No File Addresses Were Returned")
lib.PrintError("This is likely a mistyped, but valid, URL in the input.")
lib.PrintError("This also occurs if a github repo link is provided when the profile option is enabled, or vice versa")
exit()
for addr in set(file_addresses):
urlpage = connect(addr)
executor.submit(better_search_execute(displaymode, urlpage, repo_crawl, verbosity))
sleep(limiter)
lib.PrintSuccess("Scanning complete.")
else:
while True:
url_file = input("Enter the full path to the input file: ")
if isfile(url_file) is True:
break
elif str(url_file) == "":
pass
else:
lib.PrintError("No Such File Found.")
continue
with open(url_file) as ufile:
count = 0
for line in ufile.readlines():
if repo_crawl is False:
count += 1
urlpage = connect(line.rstrip())
if urlpage == 'connection failed':
lib.PrintFailure(f"[Line: {count}] Connection failed on host {line}")
else:
better_search_execute(displaymode, urlpage, repo_crawl, verbosity)
sleep(limiter)
else:
if link_type == 'profile':
resources = get_repos(line)
elif link_type == 'repo':
resources = line
file_addresses = traverse_repos(resources, link_type, directory_filtering, blacklisted_directories, verbosity)
executor = ThreadPoolExecutor(max_workers=len(file_addresses))
for addr in set(file_addresses):
urlpage = connect(addr)
executor.submit(better_search_execute(displaymode, urlpage, repo_crawl, verbosity))
sleep(limiter)
