def test_user_agent_must_be_set_or_raise(self, monkeypatch): """ The HIBP backend requires a User-Agent; ensure we're forcing one to be set on all functions """ monkeypatch.setitem(pyhibp.pyHIBP_HEADERS, 'User-Agent', None) with pytest.raises(RuntimeError) as execinfo: pyhibp.get_pastes(email_address=TEST_ACCOUNT) assert "The User-Agent must be set. Call pyhibp.set_user_agent(ua=your_agent_string) first." in str( execinfo.value)
def test_get_pastes(self): # get_pastes(email_address=TEST_ACCOUNT): resp = pyhibp.get_pastes(email_address=TEST_ACCOUNT) # The return value is a list, containing multiple dicts (1 or more) assert isinstance(resp, list) for item in resp: assert isinstance(item, dict)
def test_get_pastes_return_false_if_no_account(self): print(pyhibp.pyHIBP_HEADERS) # get_pastes(email_address=TEST_ACCOUNT): resp = pyhibp.get_pastes(email_address=TEST_NONEXISTENT_ACCOUNT_NAME + "@example.invalid") assert not resp assert isinstance(resp, list)
def combinedfile(breach_file, unique_emails, resume): config.set("settings", "combinedfilename", breach_file) config.set("settings", "completedemails", "0") if resume == False: config.set("settings", "totalemailsscanned", "0") config.write(open("settings.conf", "w")) if resume == False: with open(breach_file, 'w') as f: f.write(f"Emails,Breaches,Breach Informations,Paste Information\n") f.close() i = 0 for email in unique_emails: i += 1 e = email time.sleep(delay) resp = pyhibp.get_account_breaches(account=e, truncate_response=True) time.sleep(delay) pastes = pyhibp.get_pastes(email_address=e) breaches = f"Found in {len(resp)} Breaches" if resp: Breach_Informations = str(resp)[:32700] pas = f"Found in {len(pastes)} Pastes" with open(breach_file, 'a', encoding='UTF-8') as f: string = f"""{e},{breaches},"{Breach_Informations}",{pas}\n""" f.write(string) time.sleep(delay) per = percentage(i, len(unique_emails)) print(f"{per}% Completed") if resume: config.set("settings", "totalemailsscanned", str(i)) config.write(open("settings.conf", "w")) else: resume_count = int(config['settings']['totalemailsscanned']) config.set("settings", "totalemailsscanned", str(resume_count + 1)) config.write(open("settings.conf", "w")) config.set("settings", "completedemails", "1") config.write(open("settings.conf", "w"))
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) try: pyhibp.set_api_key(key=Load_Configuration()) except: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to set API key, make sure it is set in the configuration file.") if self.Type == "email": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_pastes(email_address=Query) logging.info(Query_Response) if Query_Response: Current_Domain = Query_Response[0]["Source"] ID = Query_Response[0]["Id"] Link = f"https://www.{Current_Domain}.com/{ID}" JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, JSON_Query_Response, "email", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_1, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "breach": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_single_breach(breach_name=Query) if Query_Response: Current_Domain = Query_Response["Domain"] Link = f"https://www.{Current_Domain}.com/" JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "breach", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_2, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "password": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pw.is_password_breached(password=Query) logging.info(Query_Response) if Query_Response: Link = f"https://{self.Domain}/Passwords?{Query}" if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, str(Query_Response), "password", ".txt") if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_2, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "account": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_account_breaches(account=Query, truncate_response=True) if Query_Response: Current_Step = 0 for Response in Query_Response: Current_Response = pyhibp.get_single_breach(breach_name=Response['Name']) JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() Link = "https://" + Current_Response['self.Domain'] if Current_Response['self.Domain'] not in Cached_Data and Current_Response['self.Domain'] not in Data_to_Cache and Current_Step < int(self.Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "account", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, Current_Response['self.Domain'], self.Result_Type_1, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Current_Response['self.Domain']) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Current_Step += 1 Cached_Data_Object.Write_Cache(Data_to_Cache) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid Type provided.") except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID, Type_of_Query, **kwargs): Data_to_Cache = [] Cached_Data = [] try: if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) try: pyhibp.set_api_key(key=Load_Configuration()) except: logging.warning(General.Date() + " Failed to set API key, make sure it is set in the configuration file.") Query_List = General.Convert_to_List(Query_List) if Type_of_Query == "email": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pyhibp.get_pastes(email_address=Query) logging.info(Query_Response) if Query_Response: Domain = Query_Response[0]["Source"] ID = Query_Response[0]["Id"] Link = "https://www." + Domain + ".com/" + ID JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True) if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, JSON_Query_Response, "email", The_File_Extension) if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Link) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") elif Type_of_Query == "breach": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pyhibp.get_single_breach(breach_name=Query) if Query_Response: Domain = Query_Response["Domain"] Link = "https://www." + Domain + ".com/" JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True) if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "breach", The_File_Extension) if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Link) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") elif Type_of_Query == "password": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pw.is_password_breached(password=Query) logging.info(Query_Response) if Query_Response: Link = "https://haveibeenpwned.com/Passwords?" + Query if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, str(Query_Response), "password", ".txt") if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Link) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") elif Type_of_Query == "account": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pyhibp.get_account_breaches(account=Query, truncate_response=True) if Query_Response: Current_Step = 0 for Response in Query_Response: Current_Response = pyhibp.get_single_breach(breach_name=Response['Name']) JSON_Query_Response = json.dumps(Current_Response, indent=4, sort_keys=True) Link = "https://" + Current_Response['Domain'] if Current_Response['Domain'] not in Cached_Data and Current_Response['Domain'] not in Data_to_Cache and Current_Step < int(Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "account", The_File_Extension) if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, Current_Response['Domain'], "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Current_Response['Domain']) Current_Step += 1 if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") else: logging.warning(General.Date() + " Invalid type provided.") except: logging.warning(General.Date() + " Execution error.")
def individual(files, resume): config.set("settings", "completedcsv", "0") if resume == False: config.set("settings", "totalcsvscanned", "0") config.write(open("settings.conf", "w")) files = files csvindex = 0 for fi in files: print(f'\nNow Scanning : {fi}') csvindex += 1 with open(fi, 'r') as f: data_frame = f.readlines() f.close() emails = re.findall(r"[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+", str(data_frame)) #emails = list(dict.fromkeys(emails)) columns = data_frame[0] breach_file = fi + '_breaches.csv' with open(breach_file, 'w') as f: f.write("Emails,Breaches,Breach Information,Paste Information," + columns) f.close() already_scanned = [] i = 0 for e in emails: i += 1 if e in already_scanned: pass else: print(f' +{e}') already_scanned.append(e) time.sleep(delay) resp = pyhibp.get_account_breaches(account=e, truncate_response=True) time.sleep(delay) pastes = pyhibp.get_pastes(email_address=e) if resp: breaches = f"Found in {len(resp)} Breaches" Breach_Informations = str(resp)[:32700] try: lines = [] for i in range(len(data_frame)): if e in data_frame[i]: lines.append(data_frame[i]) other_informations = lines[0].replace('\n', '') except Exception: other_informations = "" pas = f"Found in {len(pastes)} Pastes" with open(breach_file, 'a', encoding='UTF-8') as f: string = f"""{e},{breaches},"{Breach_Informations}",{pas},{other_informations}\n""" f.write(string) f.close() time.sleep(delay) if resume: config.set("settings", "totalcsvscanned", str(csvindex)) config.write(open("settings.conf", "w")) else: resume_count = int(config['settings']['totalcsvscanned']) config.set("settings", "totalcsvscanned", str(resume_count + 1)) config.write(open("settings.conf", "w")) per = percentage(csvindex, len(files)) print(f"{per}% Completed") config.set("settings", "completedcsv", "1") config.write(open("settings.conf", "w"))
def test_get_pastes_raise_if_email_not_string(self): # get_pastes(email_address=1): with pytest.raises(AttributeError) as excinfo: pyhibp.get_pastes(email_address=1) assert "The email address supplied must be provided, and be a string" in str( excinfo.value)
def Search(Query_List, Task_ID, Type_of_Query, **kwargs): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) try: pyhibp.set_api_key(key=Load_Configuration()) except: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to set API key, make sure it is set in the configuration file." ) Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(kwargs) if Type_of_Query == "email": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) for Query in Query_List: Query_Response = pyhibp.get_pastes(email_address=Query) logging.info(Query_Response) if Query_Response: Current_Domain = Query_Response[0]["Source"] ID = Query_Response[0]["Id"] Link = f"https://www.{Current_Domain}.com/{ID}" JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True) if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, JSON_Query_Response, "email", The_File_Extension) if Output_file: Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Account", Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name) elif Type_of_Query == "breach": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) for Query in Query_List: Query_Response = pyhibp.get_single_breach(breach_name=Query) if Query_Response: Current_Domain = Query_Response["Domain"] Link = f"https://www.{Current_Domain}.com/" JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True) if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, JSON_Query_Response, "breach", The_File_Extension) if Output_file: Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Credentials", Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name) elif Type_of_Query == "password": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) for Query in Query_List: Query_Response = pw.is_password_breached(password=Query) logging.info(Query_Response) if Query_Response: Link = f"https://{Domain}/Passwords?{Query}" if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, str(Query_Response), "password", ".txt") if Output_file: Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Credentials", Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name) elif Type_of_Query == "account": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) for Query in Query_List: Query_Response = pyhibp.get_account_breaches( account=Query, truncate_response=True) if Query_Response: Current_Step = 0 for Response in Query_Response: Current_Response = pyhibp.get_single_breach( breach_name=Response['Name']) JSON_Query_Response = json.dumps(Current_Response, indent=4, sort_keys=True) Link = "https://" + Current_Response['Domain'] if Current_Response[ 'Domain'] not in Cached_Data and Current_Response[ 'Domain'] not in Data_to_Cache and Current_Step < int( Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, JSON_Query_Response, "account", The_File_Extension) if Output_file: Output_Connections = General.Connections( Query, Local_Plugin_Name, Current_Response['Domain'], "Account", Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output( [Output_file], Link, General.Get_Title(Link), Concat_Plugin_Name) Data_to_Cache.append( Current_Response['Domain']) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Invalid type provided." ) except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def test_get_pastes_api_key_must_be_specified_or_raise(self): with pytest.raises(RuntimeError) as excinfo: # Will raise because an API key has not been set on this request. pyhibp.get_pastes(email_address=TEST_ACCOUNT) assert "A HIBP API key is required for this call. Call pyhibp.set_api_key(key=your_key) first." in str( excinfo.value)