def make_request(self, url, cf=False, meth="GET", timeout=30, redirs=True, data=None, params=None): if cf is False: try: response = requests.request(url=url, headers=self.headers, method=meth, timeout=timeout, allow_redirects=redirs, data=data, params=params) ui.debug(ui.purple, "REQUEST", response.url, ui.bold, response.status_code) except Exception as ex: ui.error("Request could not be made for", self.email, ex) else: # cf is True try: import cfscrape scraper = cfscrape.create_scraper() response = scraper.get(url) except Exception as ex: ui.error("Cloudflare bypass request could not be made for", self.email, ex) if response.status_code == 429: ui.warning("Reached RATE LIMIT, sleeping", ui.purple, self.email) sleep(2.5) return response
def main(user_args): targets = [] api_keys = get_config_from_file(user_args) ui.info_section("\n", ui.darkteal, "Targets") user_stdin_target = fetch_emails(args.target_emails) if user_stdin_target: targets.append(user_stdin_target) elif os.path.isfile(user_args.target_emails): ui.debug(ui.darkgray, "Reading from file", user_args.target_emails) targets.extend(get_emails_from_file(user_args.target_emails)) else: ui.warning("No targets found") # Launch if not user_args.run_local: breached_targets = target_factory(targets, api_keys) elif user_args.run_local: breached_targets = [Target(t) for t in targets] if user_args.bc_path: breached_targets = breachcomp_check(breached_targets, user_args.bc_path) print_results(breached_targets) if user_args.output_file: save_results_csv(user_args.output_file, breached_targets)
def init_manifest(self, manifest_url, *, branch="master", tag=None): if self.manifest_clone_path.exists(): ui.warning("Re-initializing worktree") tsrc.git.run_git(self.manifest_clone_path, "remote", "set-url", "origin", manifest_url) tsrc.git.run_git(self.manifest_clone_path, "fetch") tsrc.git.run_git(self.manifest_clone_path, "checkout", "-B", branch) tsrc.git.run_git(self.manifest_clone_path, "branch", branch, "--set-upstream-to", "origin/%s" % branch) if tag: ref = tag else: ref = "origin/%s" % branch tsrc.git.run_git(self.manifest_clone_path, "reset", "--hard", ref) else: parent, name = self.manifest_clone_path.splitpath() parent.makedirs_p() tsrc.git.run_git(self.manifest_clone_path.parent, "clone", manifest_url, name, "--branch", branch) if tag: tsrc.git.run_git(self.manifest_clone_path, "reset", "--hard", tag) self.load_manifest()
def do_transfer(self): """Performs the real transfer""" errors = dict() progress = Progress() total_start = time.time() progress.total_done = 0 progress.total_size = self.transfer_info.size progress.index = 0 progress.count = len(self.transfer_info.to_transfer) self.progress_indicator.on_start() def on_file_transfer(transfered): progress.file_done += transfered progress.total_done += transfered now = time.time() progress.total_elapsed = now - total_start progress.file_elapsed = now - file_start self.last_progress = progress if now - self.last_progress_update > 0.1: self.progress_indicator.on_progress(progress) self.last_progress_update = now for (src, dest, file_size) in self.transfer_info.to_transfer: file_start = time.time() progress.index += 1 progress.src = src progress.dest = dest progress.file_size = file_size progress.file_start = time.time() progress.file_done = 0 ftm = FileTransferManager(src, dest, self.options) ftm.set_callback(on_file_transfer) self.progress_indicator.on_new_file(progress) error = ftm.do_transfer() if self.last_progress: self.progress_indicator.on_progress(self.last_progress) self.progress_indicator.on_file_done() if error: errors[src] = error self.progress_indicator.on_finish() if self.options.move and not self.options.ignore_errors: for to_remove in self.transfer_info.to_remove: try: os.rmdir(to_remove) except OSError as error: ui.warning("Failed to remove ", to_remove, ":\n", error, end="\n", sep="") return errors
def build(self, configuration): evaluated_artefacts = self.common_parameters.artefacts.eval() evaluated_prerequisites = self.common_parameters.prerequisites.eval() if not evaluated_artefacts or not evaluated_prerequisites: ui.warning(("target {} has no artifacts or prerequisites defined, " "this means, whatever is defined in run_before or run_after " "will be always executed") .format(self.common_parameters.name)) ui.debug("phony build")
def build(self, configuration): evaluated_artefacts = self.common_parameters.artefacts.eval() evaluated_prerequisites = self.common_parameters.prerequisites.eval() if not evaluated_artefacts or not evaluated_prerequisites: ui.warning( ("target {} has no artifacts or prerequisites defined, " "this means, whatever is defined in run_before or run_after " "will be always executed").format( self.common_parameters.name)) ui.debug("phony build")
def wrapped(args=None): colored_traceback.add_hook() try: main_func(args=args) except tsrc.Error as e: # "expected" failure, display it and exit if e.message: ui.error(e.message) sys.exit(1) except KeyboardInterrupt: ui.warning("Interrupted by user, quitting") sys.exit(1)
def get_pastebin(self): try: ui.debug(self.email, "Getting pastebin private data") url = "https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=filtered_cse&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&cx=013305635491195529773:0ufpuq-fpt0&q=%22{email}%22&cse_tok=AF14hljK02r_xNhCjU6rz69O3mWFxQgr0w:1534517796222&sort=&googlehost=www.google.com".format( email=self.email) req = self.make_request(url, cf=False) response = req.json() for e in response["results"]: self.pastebin_urls.append(e["formattedUrl"]) except Exception as ex: ui.warning(ui.yellow, "Google api error for pastebin.com:", self.email, ex, url)
def get_hunterio_private(self, api_key): try: ui.debug(self.email, "Getting HunterIO private data on domain") url = "https://api.hunter.io/v2/domain-search?domain={target}&api_key={key}".format( target=self.hostname, key=api_key) req = self.make_request(url, cf=True) response = req.json() for e in response["data"]["emails"]: self.hunterio_mails.append(e["value"]) except Exception as ex: ui.warning(ui.yellow, "HunterIO (private API) error:", self.email, ex, url)
def get_hunterio_public(self): try: ui.debug(self.email, "Getting HunterIO public data on domain") url = "https://api.hunter.io/v2/email-count?domain={}".format( self.hostname) req = self.make_request(url, cf=True) response = req.json() if response["data"]["total"] != 0: self.related_emails = response["data"]["total"] except Exception as ex: ui.warning(ui.yellow, "HunterIO (pubic API) error:", self.email, ex)
def transfer_file(self): """Transfer src to dest, calling callback(transfered) while doing so, where transfered is the size of the buffer successfully transfered src and dest must be two valid file paths. If move is True, remove src when done. """ check_same_file(self.src, self.dest) if os.path.islink(self.src): handle_symlink(self.src, self.dest) self.callback(0) return src_file, dest_file = open_files(self.src, self.dest) transfered = 0 try: while True: data = src_file.read(BUFFER_SIZE) if not data: self.callback(0) break transfered = len(data) self.callback(transfered) dest_file.write(data) except IOError as err: mess = "Problem when transferring %s to %s\n" % (self.src, self.dest) mess += "Error was: %s" % err raise TransferError(mess) finally: src_file.close() dest_file.close() try: self.post_transfer() except OSError as err: ui.warning("Failed to finalize transfer of %s: %s" % (self.dest, err)) if self.options.move: try: os.remove(self.src) except OSError: ui.warning("Could not remove %s" % self.src)
def get_shodan(self, api_key): try: self.ip = socket.gethostbyname(self.hostname) except Exception as ex: ui.debug("Could not fetch host IP address", self.hostname, ex) if len(api_key) == 0: ui.debug(self.email, "Setting default Shodan API KEY") api_key = "UNmOjxeFS2mPA3kmzm1sZwC0XjaTTksy" # From Infoga tool try: url = "https://api.shodan.io/shodan/host/{target}?key={key}".format( target=self.ip, key=api_key) response = self.make_request(url) data = json.loads(response.text) self.rev_ports.extend(data["ports"]) self.rev_dns.extend(data["hostnames"]) except Exception as ex: ui.warning(ui.yellow, "Shodan error for:", self.email, ex)
def breachcomp_check(targets, breachcomp_path): # https://gist.github.com/scottlinux/9a3b11257ac575e4f71de811322ce6b3 try: import subprocess query_bin = os.path.join(breachcomp_path, "query.sh") subprocess.call(["chmod", "+x", query_bin]) for t in targets: procfd = subprocess.run([query_bin, t.email], stdout=subprocess.PIPE) output = procfd.stdout.decode("utf-8") if len(output) != 0: t.pwnd = True split_output = output.split("\n") for line in split_output: if line: t.breachcomp_passw.append(line.split(":")[1]) return targets except Exception as ex: ui.warning("Breach compilation", ex)
def get_hibp(self): sleep(1.3) url = "https://haveibeenpwned.com/api/v2/breachedaccount/{}?truncateResponse=true".format( self.email) response = self.make_request(url) if response.status_code not in [200, 404]: ui.warning("Retrying HIBP with CF bypass for", self.email) response = self.make_request(url, cf=True) if response.status_code not in [200, 404]: ui.warning("Could not contact HIBP using CF bypass for", self.email) return if response.status_code == 200: self.pwnd = True data = response.json() for d in data: # Returned type is a dict of Name : Service for name, ser in d.items(): self.services["hibp"].append(ser) ui.debug(self.services["hibp"]) elif response.status_code == 404: ui.debug(ui.bold, "HIBP:", ui.reset, ui.cross, "No breaches found for {}".format(self.email)) self.pwnd = False else: ui.warning("HIBP: got API response code", response.status_code, "for", self.email) self.pwnd = False ui.debug( ui.bold, "HIBP:", ui.reset, "{} breaches found for {}".format(len(self.services["hibp"]), self.email))
def get_snusbase(self, api_url, api_key): try: ui.debug(self.email, "Getting snusbase data") url = api_url self.headers.update({"Authorization": api_key}) payload = {"type": "email", "term": self.email} req = self.make_request(url, cf=False, meth="POST", data=payload) response = req.json() for result in response["result"]: if result["password"]: ui.debug(self.email, ":", result["password"]) self.snusbase_passw.append(result["password"]) if result["hash"]: ui.debug(self.email, ": hash found") self.snusbase_hash_salt.update( {result["hash"]: result["salt"]}) if result["tablenr"]: if result["tablenr"] not in self.services["snusbase"]: self.services["snusbase"].append(result["tablenr"]) except Exception as ex: ui.warning(ui.yellow, "Snusbase error:", self.email, ex)
def get_weleakinfo_public(self): try: url = "https://api.weleakinfo.com/v2/public/email/{}".format( self.email) req = self.make_request(url, cf=True) response = req.json() if type( response["Unique"] ) == str: # if no breach returns an int(0) else str('X Websites') for d in response["Data"]: self.services["weleakinfo"].append(d) ui.debug( ui.bold, "WeLeakInfo:", ui.reset, "{} breaches found for {}".format( len(self.services["weleakinfo"]), self.email)) else: ui.debug(ui.bold, "WeLeakInfo:", ui.reset, ui.cross, "No breaches found for {}".format(self.email)) return except Exception as ex: ui.warning(ui.yellow, "WeLeakInfo (pubic API) error:", self.email, ex)
def handle_overwrite(self): """Return True if we should skip the file. Ask user for confirmation if we were called with an 'interactive' option. """ # Safe: always skip if self.options.safe: ui.warning("Skipping", self.dest) return True # Not safe and not interactive => overwrite if not self.options.interactive: return False # Interactive print("File: '%s' already exists" % self.dest) print("Overwrite?") user_input = input() if user_input == "y": return False else: return True
def save_results_csv(dest_csv, target_obj_list): with open(dest_csv, 'w', newline='') as csvfile: writer = csv.writer(csvfile) writer.writerow([ "email", "breached", "num services", "hibp_services", "weleakinfo_services", "snusbase_services", "ip", "ports", "rev_dns", "related_emails", "snusbase_passwords", "snusbase_hash/salt", "breachcompilation_passwords" ]) print("* Writing to CSV\n") for target in target_obj_list: try: writer.writerow([ target.email, target.pwnd, len(target.services["hibp"]), target.services["hibp"], target.services["weleakinfo"], target.services["snusbase"], target.ip, target.rev_ports, target.rev_dns, target.related_emails, target.snusbase_passw, target.snusbase_hash_salt, target.breachcomp_passw ]) except Exception as ex: ui.warning("Error writing to csv", ex)
def ask_table(db_file=TWEETS_DB, conn=create_conection(), table_name=TABLE_NAME): """Default table or new table. Checks if table exists, creates one if not with name from user and calls user_ans_tweets(). :param db_file: DEFAULT database :param conn: creates connection() :param table_name: name of table """ def ploting(tweets_db=TWEETS_DB, csv_file=CSV_FILE): table_name = MyStreamListener.table_name plot_question = ui.ask_yes_no("Plot?", default=True) if plot_question: ui.info(ui.green, "Populating csv file with {}".format(table_name)) q_four.db_into_csv(TWEETS_DB, CSV_FILE, table_name) ui.info_3("Ploting...") q_four.frecuency() q_four.senti() else: ui.info(ui.turquoise, "Program Finished") exit() tables_list = list() conn = create_conection() cur = conn.cursor() cur.execute("SELECT name FROM sqlite_master") for table in cur: tables = str(table).strip('(').strip(')').strip(',').strip("'") tables_list.append(tables) if "trump" in tables_list: ui.info_1(ui.yellow, ui.bold, "DEFAULT TABLE FOUND! =>", ui.blue, ui.standout, "'trump'") new_db_ans = ui.ask_yes_no("Add tweets to table 'trump'?", default=True) if new_db_ans: ui.warning("Creating database (If it doesn't exist)") create_conection() ui.info_2(ui.green, "Accessing Database") MyStreamListener.table_name = TABLE_NAME user_ans_tweets() ploting() else: choices = ["Create New Table", "Load table", "Plot", "Quit"] new = ui.ask_choice("Select: ", choices) if new == "Create New Table": ui.warning("Tables with the same name will not be created") new_table_name = ui.ask_string("Enter new table name:") new_table_name = new_table_name.lower() new_table_name = new_table_name.replace(" ", "_") ui.info("Table name with format:", ui.blue, new_table_name) create_ans = ui.ask_yes_no("Create?") if create_ans: tweets_table_create(create_conection(), TWEETS_DB, new_table_name) insert_new_tbl = ui.ask_yes_no( "Insert new tweets into {}?".format(new_table_name)) if insert_new_tbl: MyStreamListener.table_name = new_table_name user_ans_tweets() ploting() else: ui.info(ui.bold, ("Program Finished")) else: ui.info(ui.bols, "Program Finished") elif new == "Load table": new_table_name = ui.ask_choice("Select Table to load:", tables_list) MyStreamListener.table_name = new_table_name user_ans_tweets() ploting() elif new == "Plot": new_table_name = ui.ask_choice("Select Table to plot:", tables_list) MyStreamListener.table_name = new_table_name ploting() elif new == "Quit": ui.warning("Program Finished") exit()
****************************************************************************""" import sqlite3 import tweepy import json import os from textblob import TextBlob try: import ui except ModuleNotFoundError as e: print("Please install python-cli-ui package: pip install python-cli-ui") print("Program terminated") exit() try: import graph_module as q_four except ModuleNotFoundError as e: ui.warning("graph-module.py not found!") """============================================================================= S E C U R I T Y - T O K E N S =============================================================================== The following code was taken from a story created by Lucas Kohorst on freecodecamp.org on 8th-April-unknown. Accessed on 15-Nov-2018: https://medium.freecodecamp.org/creating-a-twitter-bot-in-python-with-tweepy-ac524157a607. It also appears on tweepy documentation by Joshua Roesslein, 2009, accessed on 21-Nov-2018: http://docs.tweepy.org/en/v3.5.0/getting_started.html ****************************************************************************""" consumer_key = 'fj5RNQTL1p9XHaYysf5uYj2NP' consumer_secret = 'XHeDxpDX7GbBe435NmsNukDMAKcSGT1uxyMrEUbTJE85QUZ2lN' access_token = '52681286-TMzzPMOyS8k8QOKtAawolyKWn1gWi3OWXbUrQHjXF' access_token_secret = 'k77qveVFaTbrXcpyyGEcZASlYNRTOVwDb8Rqho2qCNU3Y' """============================================================================ G L O B A L - C O N S T A N T S ============================================================================""" TWEETS_DB = "tweets.sqlite"