def make_request(self,
url,
cf=False,
meth="GET",
timeout=30,
redirs=True,
data=None,
params=None):
if cf is False:
try:
response = requests.request(url=url,
headers=self.headers,
method=meth,
timeout=timeout,
allow_redirects=redirs,
data=data,
params=params)
ui.debug(ui.purple, "REQUEST", response.url, ui.bold,
response.status_code)
except Exception as ex:
ui.error("Request could not be made for", self.email, ex)
else: # cf is True
try:
import cfscrape
scraper = cfscrape.create_scraper()
response = scraper.get(url)
except Exception as ex:
ui.error("Cloudflare bypass request could not be made for",
self.email, ex)
if response.status_code == 429:
ui.warning("Reached RATE LIMIT, sleeping", ui.purple, self.email)
sleep(2.5)
return response
def main(user_args):
targets = []
api_keys = get_config_from_file(user_args)
ui.info_section("\n", ui.darkteal, "Targets")
user_stdin_target = fetch_emails(args.target_emails)
if user_stdin_target:
targets.append(user_stdin_target)
elif os.path.isfile(user_args.target_emails):
ui.debug(ui.darkgray, "Reading from file", user_args.target_emails)
targets.extend(get_emails_from_file(user_args.target_emails))
else:
ui.warning("No targets found")
# Launch
if not user_args.run_local:
breached_targets = target_factory(targets, api_keys)
elif user_args.run_local:
breached_targets = [Target(t) for t in targets]
if user_args.bc_path:
breached_targets = breachcomp_check(breached_targets,
user_args.bc_path)
print_results(breached_targets)
if user_args.output_file:
save_results_csv(user_args.output_file, breached_targets)
def init_manifest(self, manifest_url, *, branch="master", tag=None):
if self.manifest_clone_path.exists():
ui.warning("Re-initializing worktree")
tsrc.git.run_git(self.manifest_clone_path, "remote", "set-url",
"origin", manifest_url)
tsrc.git.run_git(self.manifest_clone_path, "fetch")
tsrc.git.run_git(self.manifest_clone_path, "checkout", "-B",
branch)
tsrc.git.run_git(self.manifest_clone_path, "branch", branch,
"--set-upstream-to", "origin/%s" % branch)
if tag:
ref = tag
else:
ref = "origin/%s" % branch
tsrc.git.run_git(self.manifest_clone_path, "reset", "--hard", ref)
else:
parent, name = self.manifest_clone_path.splitpath()
parent.makedirs_p()
tsrc.git.run_git(self.manifest_clone_path.parent, "clone",
manifest_url, name, "--branch", branch)
if tag:
tsrc.git.run_git(self.manifest_clone_path, "reset", "--hard",
tag)
self.load_manifest()
def do_transfer(self):
"""Performs the real transfer"""
errors = dict()
progress = Progress()
total_start = time.time()
progress.total_done = 0
progress.total_size = self.transfer_info.size
progress.index = 0
progress.count = len(self.transfer_info.to_transfer)
self.progress_indicator.on_start()
def on_file_transfer(transfered):
progress.file_done += transfered
progress.total_done += transfered
now = time.time()
progress.total_elapsed = now - total_start
progress.file_elapsed = now - file_start
self.last_progress = progress
if now - self.last_progress_update > 0.1:
self.progress_indicator.on_progress(progress)
self.last_progress_update = now
for (src, dest, file_size) in self.transfer_info.to_transfer:
file_start = time.time()
progress.index += 1
progress.src = src
progress.dest = dest
progress.file_size = file_size
progress.file_start = time.time()
progress.file_done = 0
ftm = FileTransferManager(src, dest, self.options)
ftm.set_callback(on_file_transfer)
self.progress_indicator.on_new_file(progress)
error = ftm.do_transfer()
if self.last_progress:
self.progress_indicator.on_progress(self.last_progress)
self.progress_indicator.on_file_done()
if error:
errors[src] = error
self.progress_indicator.on_finish()
if self.options.move and not self.options.ignore_errors:
for to_remove in self.transfer_info.to_remove:
try:
os.rmdir(to_remove)
except OSError as error:
ui.warning("Failed to remove ",
to_remove,
":\n",
error,
end="\n",
sep="")
return errors
def build(self, configuration):
evaluated_artefacts = self.common_parameters.artefacts.eval()
evaluated_prerequisites = self.common_parameters.prerequisites.eval()
if not evaluated_artefacts or not evaluated_prerequisites:
ui.warning(("target {} has no artifacts or prerequisites defined, "
"this means, whatever is defined in run_before or run_after "
"will be always executed")
.format(self.common_parameters.name))
ui.debug("phony build")
def build(self, configuration):
evaluated_artefacts = self.common_parameters.artefacts.eval()
evaluated_prerequisites = self.common_parameters.prerequisites.eval()
if not evaluated_artefacts or not evaluated_prerequisites:
ui.warning(
("target {} has no artifacts or prerequisites defined, "
"this means, whatever is defined in run_before or run_after "
"will be always executed").format(
self.common_parameters.name))
ui.debug("phony build")
def wrapped(args=None):
colored_traceback.add_hook()
try:
main_func(args=args)
except tsrc.Error as e:
# "expected" failure, display it and exit
if e.message:
ui.error(e.message)
sys.exit(1)
except KeyboardInterrupt:
ui.warning("Interrupted by user, quitting")
sys.exit(1)
def get_pastebin(self):
try:
ui.debug(self.email, "Getting pastebin private data")
url = "https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=filtered_cse&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&cx=013305635491195529773:0ufpuq-fpt0&q=%22{email}%22&cse_tok=AF14hljK02r_xNhCjU6rz69O3mWFxQgr0w:1534517796222&sort=&googlehost=www.google.com".format(
email=self.email)
req = self.make_request(url, cf=False)
response = req.json()
for e in response["results"]:
self.pastebin_urls.append(e["formattedUrl"])
except Exception as ex:
ui.warning(ui.yellow, "Google api error for pastebin.com:",
self.email, ex, url)
def get_hunterio_private(self, api_key):
try:
ui.debug(self.email, "Getting HunterIO private data on domain")
url = "https://api.hunter.io/v2/domain-search?domain={target}&api_key={key}".format(
target=self.hostname, key=api_key)
req = self.make_request(url, cf=True)
response = req.json()
for e in response["data"]["emails"]:
self.hunterio_mails.append(e["value"])
except Exception as ex:
ui.warning(ui.yellow, "HunterIO (private API) error:", self.email,
ex, url)
def get_hunterio_public(self):
try:
ui.debug(self.email, "Getting HunterIO public data on domain")
url = "https://api.hunter.io/v2/email-count?domain={}".format(
self.hostname)
req = self.make_request(url, cf=True)
response = req.json()
if response["data"]["total"] != 0:
self.related_emails = response["data"]["total"]
except Exception as ex:
ui.warning(ui.yellow, "HunterIO (pubic API) error:", self.email,
ex)
def transfer_file(self):
"""Transfer src to dest, calling
callback(transfered) while doing so,
where transfered is the size of the buffer successfully transfered
src and dest must be two valid file paths.
If move is True, remove src when done.
"""
check_same_file(self.src, self.dest)
if os.path.islink(self.src):
handle_symlink(self.src, self.dest)
self.callback(0)
return
src_file, dest_file = open_files(self.src, self.dest)
transfered = 0
try:
while True:
data = src_file.read(BUFFER_SIZE)
if not data:
self.callback(0)
break
transfered = len(data)
self.callback(transfered)
dest_file.write(data)
except IOError as err:
mess = "Problem when transferring %s to %s\n" % (self.src,
self.dest)
mess += "Error was: %s" % err
raise TransferError(mess)
finally:
src_file.close()
dest_file.close()
try:
self.post_transfer()
except OSError as err:
ui.warning("Failed to finalize transfer of %s: %s" %
(self.dest, err))
if self.options.move:
try:
os.remove(self.src)
except OSError:
ui.warning("Could not remove %s" % self.src)
def get_shodan(self, api_key):
try:
self.ip = socket.gethostbyname(self.hostname)
except Exception as ex:
ui.debug("Could not fetch host IP address", self.hostname, ex)
if len(api_key) == 0:
ui.debug(self.email, "Setting default Shodan API KEY")
api_key = "UNmOjxeFS2mPA3kmzm1sZwC0XjaTTksy" # From Infoga tool
try:
url = "https://api.shodan.io/shodan/host/{target}?key={key}".format(
target=self.ip, key=api_key)
response = self.make_request(url)
data = json.loads(response.text)
self.rev_ports.extend(data["ports"])
self.rev_dns.extend(data["hostnames"])
except Exception as ex:
ui.warning(ui.yellow, "Shodan error for:", self.email, ex)
def breachcomp_check(targets, breachcomp_path):
# https://gist.github.com/scottlinux/9a3b11257ac575e4f71de811322ce6b3
try:
import subprocess
query_bin = os.path.join(breachcomp_path, "query.sh")
subprocess.call(["chmod", "+x", query_bin])
for t in targets:
procfd = subprocess.run([query_bin, t.email],
stdout=subprocess.PIPE)
output = procfd.stdout.decode("utf-8")
if len(output) != 0:
t.pwnd = True
split_output = output.split("\n")
for line in split_output:
if line:
t.breachcomp_passw.append(line.split(":")[1])
return targets
except Exception as ex:
ui.warning("Breach compilation", ex)
def get_hibp(self):
sleep(1.3)
url = "https://haveibeenpwned.com/api/v2/breachedaccount/{}?truncateResponse=true".format(
self.email)
response = self.make_request(url)
if response.status_code not in [200, 404]:
ui.warning("Retrying HIBP with CF bypass for", self.email)
response = self.make_request(url, cf=True)
if response.status_code not in [200, 404]:
ui.warning("Could not contact HIBP using CF bypass for",
self.email)
return
if response.status_code == 200:
self.pwnd = True
data = response.json()
for d in data: # Returned type is a dict of Name : Service
for name, ser in d.items():
self.services["hibp"].append(ser)
ui.debug(self.services["hibp"])
elif response.status_code == 404:
ui.debug(ui.bold, "HIBP:", ui.reset, ui.cross,
"No breaches found for {}".format(self.email))
self.pwnd = False
else:
ui.warning("HIBP: got API response code", response.status_code,
"for", self.email)
self.pwnd = False
ui.debug(
ui.bold, "HIBP:", ui.reset,
"{} breaches found for {}".format(len(self.services["hibp"]),
self.email))
def get_snusbase(self, api_url, api_key):
try:
ui.debug(self.email, "Getting snusbase data")
url = api_url
self.headers.update({"Authorization": api_key})
payload = {"type": "email", "term": self.email}
req = self.make_request(url, cf=False, meth="POST", data=payload)
response = req.json()
for result in response["result"]:
if result["password"]:
ui.debug(self.email, ":", result["password"])
self.snusbase_passw.append(result["password"])
if result["hash"]:
ui.debug(self.email, ": hash found")
self.snusbase_hash_salt.update(
{result["hash"]: result["salt"]})
if result["tablenr"]:
if result["tablenr"] not in self.services["snusbase"]:
self.services["snusbase"].append(result["tablenr"])
except Exception as ex:
ui.warning(ui.yellow, "Snusbase error:", self.email, ex)
def get_weleakinfo_public(self):
try:
url = "https://api.weleakinfo.com/v2/public/email/{}".format(
self.email)
req = self.make_request(url, cf=True)
response = req.json()
if type(
response["Unique"]
) == str: # if no breach returns an int(0) else str('X Websites')
for d in response["Data"]:
self.services["weleakinfo"].append(d)
ui.debug(
ui.bold, "WeLeakInfo:", ui.reset,
"{} breaches found for {}".format(
len(self.services["weleakinfo"]), self.email))
else:
ui.debug(ui.bold, "WeLeakInfo:", ui.reset, ui.cross,
"No breaches found for {}".format(self.email))
return
except Exception as ex:
ui.warning(ui.yellow, "WeLeakInfo (pubic API) error:", self.email,
ex)
def handle_overwrite(self):
"""Return True if we should skip the file.
Ask user for confirmation if we were called
with an 'interactive' option.
"""
# Safe: always skip
if self.options.safe:
ui.warning("Skipping", self.dest)
return True
# Not safe and not interactive => overwrite
if not self.options.interactive:
return False
# Interactive
print("File: '%s' already exists" % self.dest)
print("Overwrite?")
user_input = input()
if user_input == "y":
return False
else:
return True
def save_results_csv(dest_csv, target_obj_list):
with open(dest_csv, 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow([
"email", "breached", "num services", "hibp_services",
"weleakinfo_services", "snusbase_services", "ip", "ports",
"rev_dns", "related_emails", "snusbase_passwords",
"snusbase_hash/salt", "breachcompilation_passwords"
])
print("* Writing to CSV\n")
for target in target_obj_list:
try:
writer.writerow([
target.email, target.pwnd,
len(target.services["hibp"]), target.services["hibp"],
target.services["weleakinfo"], target.services["snusbase"],
target.ip, target.rev_ports, target.rev_dns,
target.related_emails, target.snusbase_passw,
target.snusbase_hash_salt, target.breachcomp_passw
])
except Exception as ex:
ui.warning("Error writing to csv", ex)
def ask_table(db_file=TWEETS_DB,
conn=create_conection(),
table_name=TABLE_NAME):
"""Default table or new table. Checks if table exists,
creates one if not with name from user and calls user_ans_tweets().
:param db_file: DEFAULT database
:param conn: creates connection()
:param table_name: name of table
"""
def ploting(tweets_db=TWEETS_DB, csv_file=CSV_FILE):
table_name = MyStreamListener.table_name
plot_question = ui.ask_yes_no("Plot?", default=True)
if plot_question:
ui.info(ui.green, "Populating csv file with {}".format(table_name))
q_four.db_into_csv(TWEETS_DB, CSV_FILE, table_name)
ui.info_3("Ploting...")
q_four.frecuency()
q_four.senti()
else:
ui.info(ui.turquoise, "Program Finished")
exit()
tables_list = list()
conn = create_conection()
cur = conn.cursor()
cur.execute("SELECT name FROM sqlite_master")
for table in cur:
tables = str(table).strip('(').strip(')').strip(',').strip("'")
tables_list.append(tables)
if "trump" in tables_list:
ui.info_1(ui.yellow, ui.bold, "DEFAULT TABLE FOUND! =>", ui.blue,
ui.standout, "'trump'")
new_db_ans = ui.ask_yes_no("Add tweets to table 'trump'?",
default=True)
if new_db_ans:
ui.warning("Creating database (If it doesn't exist)")
create_conection()
ui.info_2(ui.green, "Accessing Database")
MyStreamListener.table_name = TABLE_NAME
user_ans_tweets()
ploting()
else:
choices = ["Create New Table", "Load table", "Plot", "Quit"]
new = ui.ask_choice("Select: ", choices)
if new == "Create New Table":
ui.warning("Tables with the same name will not be created")
new_table_name = ui.ask_string("Enter new table name:")
new_table_name = new_table_name.lower()
new_table_name = new_table_name.replace(" ", "_")
ui.info("Table name with format:", ui.blue, new_table_name)
create_ans = ui.ask_yes_no("Create?")
if create_ans:
tweets_table_create(create_conection(), TWEETS_DB,
new_table_name)
insert_new_tbl = ui.ask_yes_no(
"Insert new tweets into {}?".format(new_table_name))
if insert_new_tbl:
MyStreamListener.table_name = new_table_name
user_ans_tweets()
ploting()
else:
ui.info(ui.bold, ("Program Finished"))
else:
ui.info(ui.bols, "Program Finished")
elif new == "Load table":
new_table_name = ui.ask_choice("Select Table to load:",
tables_list)
MyStreamListener.table_name = new_table_name
user_ans_tweets()
ploting()
elif new == "Plot":
new_table_name = ui.ask_choice("Select Table to plot:",
tables_list)
MyStreamListener.table_name = new_table_name
ploting()
elif new == "Quit":
ui.warning("Program Finished")
exit()
****************************************************************************"""
import sqlite3
import tweepy
import json
import os
from textblob import TextBlob
try:
import ui
except ModuleNotFoundError as e:
print("Please install python-cli-ui package: pip install python-cli-ui")
print("Program terminated")
exit()
try:
import graph_module as q_four
except ModuleNotFoundError as e:
ui.warning("graph-module.py not found!")
"""=============================================================================
S E C U R I T Y - T O K E N S
===============================================================================
The following code was taken from a story created by Lucas Kohorst on freecodecamp.org on 8th-April-unknown. Accessed on 15-Nov-2018: https://medium.freecodecamp.org/creating-a-twitter-bot-in-python-with-tweepy-ac524157a607. It also appears on tweepy documentation by Joshua Roesslein, 2009, accessed on 21-Nov-2018: http://docs.tweepy.org/en/v3.5.0/getting_started.html
****************************************************************************"""
consumer_key = 'fj5RNQTL1p9XHaYysf5uYj2NP'
consumer_secret = 'XHeDxpDX7GbBe435NmsNukDMAKcSGT1uxyMrEUbTJE85QUZ2lN'
access_token = '52681286-TMzzPMOyS8k8QOKtAawolyKWn1gWi3OWXbUrQHjXF'
access_token_secret = 'k77qveVFaTbrXcpyyGEcZASlYNRTOVwDb8Rqho2qCNU3Y'
"""============================================================================
G L O B A L - C O N S T A N T S
============================================================================"""
TWEETS_DB = "tweets.sqlite"
