def scrape(scrape_input_method, displaymode, limiter, repo_crawl, verbosity): if scrape_input_method.lower() == 'm': url = input("Enter the URL: ") urlpage = connect(url) if urlpage == 'connection failed': lib.PrintError( "Connection to specified URL could not be established.") exit() else: lib.PrintStatus('Status: [200], Searching for API Keys...') if repo_crawl is False: search_execute(displaymode, urlpage) else: repository_list = get_repos(url) file_addresses = traverse_repos(repository_list, verbosity) executor = ThreadPoolExecutor(max_workers=len(file_addresses)) for addr in set(file_addresses): urlpage = connect(addr) executor.submit(search_execute(displaymode, urlpage)) sleep(limiter) lib.PrintSuccess("Scanning complete.") else: while True: url_file = input("Enter the full path to the input file: ") if isfile(url_file) is True: break elif str(url_file) == "": lib.DoNothing() else: lib.PrintError("No Such File Found.") continue with open(url_file) as ufile: count = 0 for line in ufile.readlines(): if repo_crawl is False: count += 1 urlpage = connect(line.rstrip()) if urlpage == 'connection failed': lib.PrintFailure( f"[Line: {count}] Connection failed on host {line}" ) else: search_execute(displaymode, urlpage) sleep(limiter) else: repository_list = get_repos(line) file_addresses = traverse_repos(repository_list, verbosity) executor = ThreadPoolExecutor( max_workers=len(file_addresses)) for addr in set(file_addresses): urlpage = connect(addr) executor.submit(search_execute(displaymode, urlpage)) sleep(limiter)
def main(): lib.PrintTitle() while True: conf = input("[L]oad configuration file or do [m]anual setup: ") if conf not in ['l', 'm']: lib.PrintError("Invalid Input") continue elif conf.lower() == 'l': vars_dict = load_config() if vars_dict['search_type'] == 's': shodan_scan(vars_dict['shodan_key'], vars_dict['local_dir']) lib.PrintSuccess("Scan Complete") break elif vars_dict['search_type'] == 'f': file_scan(vars_dict['address_file'], vars_dict['local_dir']) lib.PrintSuccess("Scan Complete") break else: masscan_scan(vars_dict['local_dir']) lib.PrintSuccess("Scan Complete") break else: vars_dict = manual_setup() if vars_dict['search_type'] == 's': shodan_scan(vars_dict['shodan_key'], vars_dict['local_dir']) lib.PrintSuccess("Scan Complete") break elif vars_dict['search_type'] == 'f': file_scan(vars_dict['address_file'], vars_dict['local_dir']) lib.PrintSuccess("Scan Complete") break else: masscan_scan(vars_dict['local_dir']) lib.PrintSuccess("Scan Complete") break
def redis_search(displaymode, page, repo_crawl, verbosity): if repo_crawl is False: lib.PrintStatus("Scanning for Redis secrets...") elif repo_crawl is True and verbosity == 'on': lib.PrintStatus("Scanning for Redis secrets...") pagetext = page.text redis_pattern = r'redis://[0-9a-zA-Z:@.\\-]+' redis_artifacts = ['REDIS_PASSWORD', 'REDIS_CACHE_DATABASE', 'REDIS_HOST', 'REDIS_DATABASE'] for k in re.findall(redis_pattern, pagetext): if displaymode == 's' or 'b': lib.PrintHighSeverity('Warning: High Severity Item Found') redis_output = f'{curdir}/Output/Redis/RedisLinks.txt' if not exists(dirname(redis_output)): try: makedirs(dirname(redis_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(redis_output, 'a') as gofile: gofile.write(f'Potential link: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential link: {k}') lib.PrintHighSeverity('Warning: High Severity Item Found') for ra in set(redis_artifacts): if ra in pagetext: lib.PrintHighSeverity('Warning: High Severity Item Found') if displaymode == 's' or 'b': redis_artifacts_output = f'{curdir}/Output/Redis/RedisArtifacts.txt' if not exists(dirname(redis_artifacts_output)): try: makedirs(dirname(redis_artifacts_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(redis_artifacts_output, 'a') as rafile: rafile.write(f'Artifact found: {ra}') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Artifact Found: {ra}')
def ftp_operations(host, output_location): # TODO: Find out how to check and record file sizes in relation to FTP if output_location.endswith('/') is False: output_location = output_location + '/' ftp_connection = ftplib.FTP(host) try: ftp_connection.login() lib.PrintSuccess('Login Status: 200') lib.PrintStatus(f'Exfiltrating files to {output_location}') filenames = ftp_connection.nlst() for filename in filenames: local_filename = path.join(output_location, filename) file = open(local_filename, 'wb') ftp_connection.retrbinary('RETR ' + filename, file.write) except Exception as e: lib.PrintError(f'{e}')
def slack_api_search(displaymode, page): lib.PrintStatus("Scanning for slack API keys...") pagetext = page.text slack_api_pattern = r"xoxp-\\d+-\\d+-\\d+-[0-9a-f]+" for k in re.findall(slack_api_pattern, pagetext): if displaymode == 's' or 'b': sapi_output = f'{curdir}\\Output\\SlackAPIPotentialKeys.txt' if not exists(dirname(sapi_output)): try: makedirs(dirname(sapi_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(sapi_output, 'a') as gofile: gofile.write(f'Potential Key: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Key: {k}')
def slack_bot_search(displaymode, page): lib.PrintStatus("Scanning for slack bot tokens...") pagetext = page.text slack_bot_pattern = r"xoxb-\\d+-[0-9a-zA-Z]+" for k in re.findall(slack_bot_pattern, pagetext): if displaymode == 's' or 'b': slack_bot_output = f'{curdir}/Output/SlackBotPotentialTokens.txt' if not exists(dirname(slack_bot_output)): try: makedirs(dirname(slack_bot_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(slack_bot_output, 'a') as gofile: gofile.write(f'Potential Token: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Token: {k}')
def nonspecific_api_search(displaymode, page): lib.PrintStatus("Scanning for nonspecific API keys...") pagetext = page.text nonspecific_pattern = r"[a|A][p|P][i|I][_]?[k|K][e|E][y|Y].{0,30}['\"\\s][0-9a-zA-Z]{32,45}['\"\\s]" for k in re.findall(nonspecific_pattern, pagetext): if displaymode == 's' or 'b': nonspecific_output = f'{curdir}/Output/NonspecificPotentialKeys.txt' if not exists(dirname(nonspecific_output)): try: makedirs(dirname(nonspecific_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(nonspecific_output, 'a') as gofile: gofile.write(f'Potential Key: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Key: {k}')
def twilio_search(displaymode, page): lib.PrintStatus("Scanning for twilio keys...") pagetext = page.text twilio_pattern = r"SK[a-z0-9]{32}" for k in re.findall(twilio_pattern, pagetext): if displaymode == 's' or 'b': twilio_output = f'{curdir}/Output/TwilioKeys.txt' if not exists(dirname(twilio_output)): try: makedirs(dirname(twilio_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(twilio_output, 'a') as gofile: gofile.write(f'Potential Key: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Key: {k}')
def github_search(displaymode, page): lib.PrintStatus("Searching for Github keys...") github_api = r"[g|G][i|I][t|T][h|H][u|U][b|B].{0,30}['\"\\s][0-9a-zA-Z]{35,40}['\"\\s]" pagetext = page.text for k in re.findall(github_api, pagetext): if displaymode == 's' or 'b': github_output = f'{curdir}/Output/GithubPotentialKeys.txt' if not exists(dirname(github_output)): try: makedirs(dirname(github_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(github_output, 'a') as gofile: gofile.write(f'Potential Key: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Key: {k}')
def heroku_search(displaymode, page): lib.PrintStatus("Scanning for Heroku API keys...") pagetext = page.text heroku_pattern = r"[h|H][e|E][r|R][o|O][k|K][u|U].{0,30}[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}" for k in re.findall(heroku_pattern, pagetext): if displaymode == 's' or 'b': heroku_output = f'{curdir}/Output/HerokuKeys.txt' if not exists(dirname(heroku_output)): try: makedirs(dirname(heroku_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(heroku_output, 'a') as gofile: gofile.write(f'Potential Key: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Key: {k}')
def discord_webhook_search(displaymode, page): lib.PrintStatus("Scanning for discord webhooks...") pagetext = page.text discord_webhook_pattern = r"(https:\/\/discordapp\.com\/api\/webhooks\/[\d]+\/[\w]+)" for k in re.findall(discord_webhook_pattern, pagetext): if displaymode == 's' or 'b': discord_webhook_output = f'{curdir}/Output/DiscordWebhooks.txt' if not exists(dirname(discord_webhook_output)): try: makedirs(dirname(discord_webhook_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(discord_webhook_output, 'a') as gofile: gofile.write(f'Potential Hook: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Hook: {k}')
def discord_nitro_search(displaymode, page): lib.PrintStatus("Scanning for discord nitro links...") pagetext = page.text discord_nitro_pattern = r"(https:\/\/discord\.gift\/.+[a-z{1,16}])" for k in re.findall(discord_nitro_pattern, pagetext): if displaymode == 's' or 'b': discord_nitro_output = f'{curdir}/Output/DiscordNitroPotentialLinks.txt' if not exists(dirname(discord_nitro_output)): try: makedirs(dirname(discord_nitro_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(discord_nitro_output, 'a') as gofile: gofile.write(f'Potential link: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential link: {k}')
def slack_webhook_search(displaymode, page): lib.PrintStatus("Scanning for slack webhooks...") pagetext = page.text slack_webhook_pattern = r"https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}" for k in re.findall(slack_webhook_pattern, pagetext): if displaymode == 's' or 'b': slack_webhook_output = f'{curdir}/Output/SlackWebhooks.txt' if not exists(dirname(slack_webhook_output)): try: makedirs(dirname(slack_webhook_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(slack_webhook_output, 'a') as gofile: gofile.write(f'Potential Hook: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Hook: {k}')
def discord_bot_search(displaymode, page): lib.PrintStatus("Scanning for discord bot tokens...") pagetext = page.text discord_token_pattern = r"([\w\-\.]+[\-\.][\w\-\.]+)" for k in re.findall(discord_token_pattern, pagetext): if displaymode == 's' or 'b': discord_bot_output = f'{curdir}/Output/DiscordBotPotentialTokens.txt' if not exists(dirname(discord_bot_output)): try: makedirs(dirname(discord_bot_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(discord_bot_output, 'a') as gofile: gofile.write(f'Potential Token: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Token: {k}')
def google_api_search(displaymode, page): lib.PrintStatus("Scanning for google API keys...") pagetext = page.text google_api_pattern = r'AIzaSy[0-9a-zA-Z_\\-]{33}' for k in re.findall(google_api_pattern, pagetext): if displaymode == 's' or 'b': gapi_output = f'{curdir}/Output/GoogleAPIPotentialKeys.txt' if not exists(dirname(gapi_output)): try: makedirs(dirname(gapi_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(gapi_output, 'a') as gofile: gofile.write(f'Potential Key: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Key: {k}')
def AWS_search(displaymode, page): lib.PrintStatus("Searching for AWS Access Keys...") aws_pattern = r"AKIA[0-9A-Z]{16}" pagetext = page.text for k in re.findall(aws_pattern, pagetext): if displaymode == 's' or 'b': aws_output = f'{curdir}/Output/AWSPotentialTokens.txt' if not exists(dirname(aws_output)): try: makedirs(dirname(aws_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(aws_output, 'a') as gofile: gofile.write(f'Potential Tokens: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Token: {k}') lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def redis_search(displaymode, page): lib.PrintStatus("Scanning for Redis URLs...") pagetext = page.text redis_pattern = r'redis://[0-9a-zA-Z:@.\\-]+' for k in re.findall(redis_pattern, pagetext): if displaymode == 's' or 'b': redis_output = f'{curdir}/Output/RedisLinks.txt' if not exists(dirname(redis_output)): try: makedirs(dirname(redis_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(redis_output, 'a') as gofile: gofile.write(f'Potential link: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential link: {k}') lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def facebook_OAUTH(displaymode, page): lib.PrintStatus("Scanning for facebook OAUTH secrets...") pagetext = page.text fauth_pattern = r"[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].{0,30}['\"\\s][0-9a-f]{32}['\"\\s]" for k in re.findall(fauth_pattern, pagetext): if displaymode == 's' or 'b': lib.PrintHighSeverity('\nWarning: High Severity Item Found\n') fauth_output = f'{curdir}/Output/FacebookOAUTHSecrets.txt' if not exists(dirname(fauth_output)): try: makedirs(dirname(fauth_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(fauth_output, 'a') as gofile: gofile.write(f'Potential Secret: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Secret: {k}') lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def ssh_keys_search(displaymode, page): lib.PrintStatus("Scanning for SSH Keys...") pagetext = page.text ssh_keys_identifiers = ["-----BEGIN OPENSSH PRIVATE KEY-----", "-----BEGIN DSA PRIVATE KEY-----", "-----BEGIN EC PRIVATE KEY-----"] for pattern in set(ssh_keys_identifiers): if pattern in pagetext: if displaymode == 's' or 'b': ssh_output = f'{curdir}/Output/SSHKeys.txt' if not exists(dirname(ssh_output)): try: makedirs(dirname(ssh_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(ssh_output, 'a') as gofile: gofile.write(f'SSH Key: {pattern}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'SSH Key: {pattern}') lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def google_oauth_search(displaymode, page): lib.PrintStatus("Scanning for google OAUTH secrets...") pagetext = page.text gauth_pattern = r"(\"client_secret\":\"[a-zA-Z0-9-_]{24}\")" for k in re.findall(gauth_pattern, pagetext): if displaymode == 's' or 'b': lib.PrintHighSeverity('\nWarning: High Severity Item Found\n') gauth_output = f'{curdir}/Output/GoogleOAUTHSecrets.txt' if not exists(dirname(gauth_output)): try: makedirs(dirname(gauth_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(gauth_output, 'a') as gofile: gofile.write(f'Potential Secret: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Secret: {k}') lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def google_access_token_search(displaymode, page): lib.PrintStatus("Scanning for google access tokens...") pagetext = page.text gat_pattern = r'ya29.[0-9a-zA-Z_\\-]{68}' for k in re.findall(gat_pattern, pagetext): if displaymode == 's' or 'b': lib.PrintHighSeverity('\nWarning: High Severity Item Found\n') gat_output = f'{curdir}/Output/GoogleAccessPotentialTokens.txt' if not exists(dirname(gat_output)): try: makedirs(dirname(gat_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(gat_output, 'a') as gofile: gofile.write(f'Potential Token: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Token: {k}') lib.PrintHighSeverity('\nWarning: High Severity Item Found\n')
def generic_search(key, displaymode, page, repo_crawl, verbosity): if repo_crawl is False: lib.PrintStatus(f"Searching for {key[0]} keys...") elif repo_crawl is True and verbosity == 'on': lib.PrintStatus(f"Searching for {key[0]} keys...") pagetext = page.text for k in re.findall(lib.patterns_dict[key], pagetext): if displaymode == 's' or 'b': if key[1] is True: lib.PrintHighSeverity("Warning: High Severity Item Found") gen_output = f'{curdir}/Output/{key[0]}.txt' if not exists(dirname(gen_output)): try: makedirs(dirname(gen_output)) except OSError as racecondition: if racecondition.errno != errno.EEXIST: raise with open(gen_output, 'a') as gofile: gofile.write(f'Potential Key: {k}\n') elif displaymode == 'p' or 'b': lib.PrintSuccess(f'Potential Key: {k}')
keylisting = parser.getboolean('initial_vars', 'keylisting') key_list = parser.get('initial_vars', 'key_list') arch_mode = parser.get('initial_vars', 'arch_mode') ArchiveSearch(stop_input, arch_mode) else: lib.PrintError("No such file found") continue elif configchoice.lower() == 'n': while True: workpath = input( "Enter the path you wish to save text documents to (enter curdir for current directory): " ) if workpath.lower() == 'curdir': workpath = curdir if path.isdir(workpath): lib.PrintSuccess("Valid Path...") if workpath.endswith('\\'): pass else: workpath = workpath + str('\\') break else: lib.PrintError("Invalid path, check input...") continue while True: try: stopinput_input = input( "Run in a constant loop? [y]/[n]: ") if stopinput_input.lower() == 'y': stop_input = True
def ArchiveSearch(stop, amode): arch_runs = 0 while True: if arch_runs > 0: lib.PrintStatus("Runs: " + str(arch_runs)) if arch_runs >= stop and stop is False: lib.PrintSuccess("Runs Complete, Operation Finished... [" + str(datetime.now().strftime('%X')) + "]") exit() else: lib.PrintStatus("Pastes fetched, cooling down for " + str(cooldown) + " seconds... [" + str(datetime.now().strftime('%X')) + "]") sleep(cooldown / 2) lib.PrintStatus("Halfway through at [" + str(datetime.now().strftime('%X')) + "]") sleep(cooldown / 2) lib.PrintStatus("resuming... [" + str(datetime.now().strftime('%X')) + "]") if arch_runs < stop or stop is True: arch_page, arch_filename = archive_connect() arch_soup = BeautifulSoup(arch_page.text, 'html.parser') sleep(2) lib.PrintStatus("Getting archived pastes... [" + str(datetime.now().strftime('%X')) + "]") if AccessDeniedError in arch_page.text: lib.PrintError( "IP Temporarily suspending, pausing until the ban is lifted. Estimated time: one hour... [" + str(datetime.now().strftime('%X')) + "]") sleep(cooldown) lib.PrintStatus("Process resumed... [" + str(datetime.now().strftime('%X')) + "]") continue else: pass lib.PrintStatus("Finding params... [" + str(datetime.now().strftime('%X')) + "]") table = arch_soup.find( "table", class_="maintable") # Fetch the table of recent pastes while True: try: tablehrefs = table.findAll( 'a', href=True) # Find the <a> tags for every paste break except AttributeError: lib.PrintError( "IP Temporarily suspending, pausing until the ban is lifted. Estimated time: one hour... [" + str(datetime.now().strftime('%X')) + "]") sleep(cooldown) lib.PrinrError("Process resumed... [" + str(datetime.now().strftime('%X')) + "]") continue for h in tablehrefs: proch = h['href'] # fetch the URL param for each paste lib.PrintSuccess("params fetched... [" + str(datetime.now().strftime('%X')) + "]") lib.PrintStatus("Acting on param " + str(proch) + "... [" + str(datetime.now().strftime('%X')) + "]") full_archpage, full_arch_url = parameter_connect(proch) sleep(5) item_soup = BeautifulSoup(full_archpage.text, 'html.parser') unprocessed = item_soup.find( 'textarea') # Fetch the raw text in the paste. for tag in taglist: unprocessed = str(unprocessed).replace( tag, "" ) # process the raw text by removing all html elements if amode == 'r': if path.isdir(workpath) is True: if blacklisting is True: flagged = False compare_text = re.sub( r'\s+', '', unprocessed ) # strip all whitespace for comparison for b in blacklist: b = re.sub( r'\s+', '', b) # strip all whitespace for comparison if b.lower() in compare_text.lower(): lib.PrintStatus( "Blacklisted phrase detected, passing..." ) flagged = True if flagged is True: continue else: arch_final_file = codecs.open( str(workpath) + str(full_arch_url).replace( ":", "-").replace(":", "-").replace( "/", "-") + ".txt", 'w+', 'utf-8') arch_final_file.write(unprocessed) arch_final_file.close() arch_runs += 1 continue elif blacklisting is False: arch_final_file = codecs.open( str(workpath) + str(full_arch_url).replace(":", "-").replace( ":", "-").replace("/", "-") + ".txt", 'w+', 'utf-8') arch_final_file.write(unprocessed) arch_final_file.close() arch_runs += 1 continue else: lib.PrintStatus("Making directory... [" + str(datetime.now().strftime('%X')) + "]") if blacklisting is True: flagged = False compare_text = re.sub( r'\s+', '', unprocessed ) # strip all whitespace for comparison for b in blacklist: b = re.sub( r'\s+', '', b) # strip all whitespace for comparison if b.lower() in compare_text.lower(): lib.PrintStatus( "Blacklisted phrase detected, passing..." ) flagged = True if flagged is True: continue else: arch_final_file = codecs.open( str(workpath) + str(full_arch_url).replace( ":", "-").replace(":", "-").replace( "/", "-") + ".txt", 'w+', 'utf-8') arch_final_file.write(unprocessed) arch_final_file.close() arch_runs += 1 continue elif blacklisting is False: arch_final_file = codecs.open( str(workpath) + str(full_arch_url).replace(":", "-").replace( ":", "-").replace("/", "-") + ".txt", 'w+', 'utf-8') arch_final_file.write(unprocessed) arch_final_file.close() arch_runs += 1 continue elif amode == 'f': if path.isdir(workpath) is True: lib.PrintStatus("Running engine... [" + str(datetime.now().strftime('%X')) + "]") if blacklisting is True: flagged = False compare_text = re.sub( r'\s+', '', unprocessed ) # strip all whitespace for comparison for b in blacklist: b = re.sub( r'\s+', '', b) # strip all whitespace for comparison if b.lower() in compare_text.lower(): lib.PrintStatus( "Blacklisted phrase detected, passing..." ) flagged = True if flagged is True: continue else: archive_engine(unprocessed, keylisting, reglisting) arch_runs += 1 continue else: lib.PrintStatus( "Running engine... [" + str(datetime.now().strftime('%X')) + "]") archive_engine(unprocessed, keylisting, reglisting) arch_runs += 1 continue else: lib.PrintSuccess("Operation Finished... [" + str(datetime.now().strftime('%X')) + "]") break
def scrape(scrape_input_method, displaymode, limiter, repo_crawl, link_type, directory_filtering, blacklisted_directories, verbosity): if scrape_input_method.lower() == 'm': url = input("Enter the URL: ") if url[len(url)-1] == ' ': url = url[:len(url)-1] urlpage = connect(url) if urlpage == 'connection failed': lib.PrintError("Connection to specified URL could not be established.") exit() else: lib.PrintStatus('Status: [200], Searching for API Keys...') if repo_crawl is False: better_search_execute(displaymode, urlpage, repo_crawl, verbosity) else: if link_type == 'profile': resources = get_repos(url) file_addresses = traverse_repos(resources, link_type, directory_filtering, blacklisted_directories, verbosity) elif link_type == 'repo': file_addresses = traverse_repos(url, link_type, directory_filtering, blacklisted_directories, verbosity) if len(file_addresses) > 0: executor = ThreadPoolExecutor(max_workers=len(file_addresses)) else: lib.PrintError("Fatal Error: No File Addresses Were Returned") lib.PrintError("This is likely a mistyped, but valid, URL in the input.") lib.PrintError("This also occurs if a github repo link is provided when the profile option is enabled, or vice versa") exit() for addr in set(file_addresses): urlpage = connect(addr) executor.submit(better_search_execute(displaymode, urlpage, repo_crawl, verbosity)) sleep(limiter) lib.PrintSuccess("Scanning complete.") else: while True: url_file = input("Enter the full path to the input file: ") if isfile(url_file) is True: break elif str(url_file) == "": pass else: lib.PrintError("No Such File Found.") continue with open(url_file) as ufile: count = 0 for line in ufile.readlines(): if repo_crawl is False: count += 1 urlpage = connect(line.rstrip()) if urlpage == 'connection failed': lib.PrintFailure(f"[Line: {count}] Connection failed on host {line}") else: better_search_execute(displaymode, urlpage, repo_crawl, verbosity) sleep(limiter) else: if link_type == 'profile': resources = get_repos(line) elif link_type == 'repo': resources = line file_addresses = traverse_repos(resources, link_type, directory_filtering, blacklisted_directories, verbosity) executor = ThreadPoolExecutor(max_workers=len(file_addresses)) for addr in set(file_addresses): urlpage = connect(addr) executor.submit(better_search_execute(displaymode, urlpage, repo_crawl, verbosity)) sleep(limiter)