def test_spinner_getters_setters(self):
"""Test spinner getters and setters.
"""
spinner = Halo()
self.assertEqual(spinner.text, '')
self.assertEqual(spinner.color, 'cyan')
self.assertIsNone(spinner.spinner_id)
spinner.spinner = 'dots12'
spinner.text = 'bar'
spinner.color = 'red'
self.assertEqual(spinner.text, 'bar')
self.assertEqual(spinner.color, 'red')
if is_supported():
self.assertEqual(spinner.spinner, Spinners['dots12'].value)
else:
self.assertEqual(spinner.spinner, default_spinner)
spinner.spinner = 'dots11'
if is_supported():
self.assertEqual(spinner.spinner, Spinners['dots11'].value)
else:
self.assertEqual(spinner.spinner, default_spinner)
spinner.spinner = 'foo_bar'
self.assertEqual(spinner.spinner, default_spinner)
# Color is None
spinner.color = None
spinner.start()
spinner.stop()
self.assertIsNone(spinner.color)
def download_files(folder_title: str, LINKS: List[str], args: argparse.Namespace):
"""
Download files when the given URL is parsed
"""
ROOT_PATH = Path.cwd()
links_len = len(LINKS)
if args.name:
dir_path = ROOT_PATH / args.name
else:
dir_path = ROOT_PATH / folder_title
# Create folder
spinner = Halo(text="Creating folder", spinner="dots")
spinner.start()
try:
dir_path.mkdir()
except FileExistsError as _:
pass
spinner.stop_and_persist(symbol="✅".encode("utf-8"), text="Folder Created")
print(f"Total files: {links_len}")
print(f"Download Path: {dir_path}")
# Start download
for index, url in enumerate(LINKS):
r = requests.get(url, stream=True, headers={"Accept-Encoding": None})
total_size = int(r.headers.get("Content-Length"))
spinner.text = f"Downloading {index + 1}/{links_len} file"
spinner.spinner = "arrow3"
spinner.start()
time.sleep(1)
spinner.stop()
file = url.split("/")[-1]
file = (file[:50] + "...") if len(file) > 50 else file
with open(dir_path / file, "wb") as f:
with tqdm(
total=total_size,
desc=f"{file:<53}",
unit="B",
unit_scale=True,
bar_format="{l_bar}{bar:20}{r_bar}{bar:-10b}",
) as pbar:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
pbar.update(len(chunk))
spinner.spinner = "monkey"
spinner.start()
time.sleep(2)
spinner.stop_and_persist(symbol="🔥".encode("utf-8"), text="All files downloaded.")
def test_spinner_getter(self):
instance = Halo()
if is_supported():
default_spinner_value = "dots"
else:
default_spinner_value = "line"
instance.spinner = default_spinner_value
self.assertEqual(default_spinner, instance.spinner)
instance.spinner = "This_spinner_do_not_exist"
self.assertEqual(default_spinner, instance.spinner)
instance.spinner = -123
self.assertEqual(default_spinner, instance.spinner)
def cli():
parser = argparse.ArgumentParser(
description=
'A web scraping tool to help pentesters crawl websites for vulnerable personal info'
)
parser.add_argument('url',
metavar='N',
type=str,
nargs='+',
help='url(s) to crawl for vulnerabilities')
parser.add_argument('-f',
'--files',
default=False,
action="store_true",
help='Download / crawl compatible file types.')
parser.add_argument(
'-l',
dest='limit',
type=int,
nargs='?',
help='Limits the amount of URLs to spider/crawl for each site')
parser.add_argument('-o',
dest='output',
type=str,
nargs='?',
help='Writes output to file path specified.')
parser.add_argument(
'-x',
dest='spider',
default=True,
action="store_false",
help='Only crawl URLs provided. Do not spider using found URLs.')
parser.add_argument('--no-relatives',
dest='relatives',
default=True,
action="store_false",
help='Disables confidence / entity processing.')
parser.add_argument(
'--debug',
dest='debug',
default=False,
action="store_true",
help=
'Debugging is actually all about finding the bug. About understanding why the bug was there to begin with. About knowing that it’s existence was no accident.'
)
args = parser.parse_args()
emails = {}
phones = {}
spinner = Halo(text='Get hyped.', spinner='dots')
spinner.start()
i = 1
nlp = None
if args.relatives:
spinner.text += ' Loading NLP library. '
nlp = spacy.load('en_core_web_md')
for base_url in args.url:
cleaned_urls = [base_url]
pbar = tqdm(cleaned_urls, position=1)
for url in pbar:
if base_url in url and file_type(url) != None:
n = len(cleaned_urls)
pbar.total = n
if args.limit:
n = args.limit
spinner.text = '{}/{} {:15} {}'.format(i, n, 'Scraping...',
url)
spinner.spinner = 'simpleDotsScrolling'
# build a beautiful soup web scraper
scraper = get_scraper(url)
if scraper:
# pull strings, urls, and body text from the current url
found_strings, found_urls = format_site(scraper)
# run website page through nlp, build and return "entities" of people / orgs
# build entity class in vuln dict if entity not already present
new_emails, new_urls, new_phones, new_ips = crawl_strings(
found_urls + found_strings)
add_pages(new_emails, url, all_vulns)
add_pages(new_phones, url, all_vulns)
if args.relatives and len(new_emails) + len(new_phones) > 1:
spinner.text = '{}/{} {:15} {}'.format(
i, n, 'Tokenizing...', url)
spinner.spinner = 'simpleDotsScrolling'
tokens = nlp(' '.join(found_strings))
spinner.text = '{}/{} {:15} {}'.format(
i, n, 'Comparing...', url)
spinner.spinner = 'squish'
vulns = find_ents(tokens)
for vuln in vulns:
if vuln.kind == 'email':
emails[vuln.name]['relatives'] = emails[
vuln.name]['relatives'].union(
vuln.relatives)
if args.spider:
cleaned_found_urls = add_protocols(
base_url, found_urls)
for found_url in cleaned_found_urls:
if found_url not in cleaned_urls and base_url in found_url:
cleaned_urls.append(found_url)
i += 1
if args.limit and i > args.limit:
break
output = print_title('Emails') + print_dict(emails)
if args.output:
f = open(args.output, 'w')
f.write(output)
spinner.succeed('All done! Output written to {}'.format(args.filepath))
else:
spinner.succeed('All done! Here you go <3')
print(output)
# -*- coding: utf-8 -*-
"""Example for doge spinner ;)
"""
from __future__ import unicode_literals
import os
import sys
import time
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from halo import Halo
spinner = Halo(text='Such Spins', spinner='dots')
try:
spinner.start()
time.sleep(2)
spinner.text = 'Much Colors'
spinner.color = 'magenta'
time.sleep(2)
spinner.text = 'Very emojis'
spinner.spinner = 'hearts'
time.sleep(2)
spinner.stop_and_persist({
'symbol': '🦄 '.encode('utf-8'),
'text': 'Wow!'
})
except (KeyboardInterrupt, SystemExit):
spinner.stop()
# -*- coding: utf-8 -*-
"""Examples for halo.
"""
from __future__ import unicode_literals, absolute_import, print_function
import os
import time
os.sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from halo import Halo
spinner = Halo({'text': 'Such Spins', 'spinner': 'dots'})
try:
spinner.start()
time.sleep(2)
spinner.text = 'Much Colors'
spinner.color = 'magenta'
time.sleep(2)
spinner.text = 'Very emojis'
spinner.spinner = {'spinner': 'hearts'}
time.sleep(2)
spinner.stop_and_persist({
'symbol': '🦄 '.encode('utf-8'),
'text': 'Wow!'
})
except (KeyboardInterrupt, SystemExit):
spinner.stop()
