def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: BSB_Search_URL = f"https://www.{self.Domain}/{Query}.html" Responses = Common.Request_Handler( BSB_Search_URL, Filter=True, Host=f"https://www.{self.Domain}") Response = Responses["Filtered"] Error_Regex = Common.Regex_Handler( Response, Custom_Regex=r"Correct\sthe\sfollowing\serrors") Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) if not Error_Regex: if BSB_Search_URL not in Cached_Data and BSB_Search_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Response, Query, self.The_File_Extension) if Output_file: Output_Connections.Output( [Output_file], BSB_Search_URL, General.Get_Title(BSB_Search_URL), self.Plugin_Name.lower()) Data_to_Cache.append(BSB_Search_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Query returned error, probably does not exist." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) Query_List = General.Convert_to_List(Query_List) for Query in Query_List: BSB_Search_URL = f"https://www.bsbnumbers.com/{Query}.html" Response = requests.get(BSB_Search_URL).text Error_Regex = re.search(r"Correct\sthe\sfollowing\serrors", Response) Output_Connections = General.Connections(Query, Plugin_Name, "bsbnumbers.com", "BSB Details", Task_ID, Plugin_Name.lower()) if not Error_Regex: if BSB_Search_URL not in Cached_Data and BSB_Search_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Response, Query, The_File_Extension) if Output_file: Output_Connections.Output( [Output_file], BSB_Search_URL, General.Get_Title(BSB_Search_URL), Plugin_Name.lower()) Data_to_Cache.append(BSB_Search_URL) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Query returned error, probably does not exist." ) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w") except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Craigslist_Location = Load_Configuration() Cached_Data = General.Get_Cache(Directory, Plugin_Name) Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(kwargs) for Query in Query_List: Main_URL = f"https://{Craigslist_Location.lower()}.craigslist.org/search/sss?format=rss&query={Query}" Craigslist_Response = feedparser.parse(Main_URL) Craigslist_Items = Craigslist_Response["items"] Current_Step = 0 for Item in Craigslist_Items: Item_URL = Item["link"] if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(Limit): Craigslist_Response = requests.get(Item_URL).text Local_URL = f"https://{Craigslist_Location.lower()}.craigslist.org/" Local_Domain = f"{Craigslist_Location.lower()}.craigslist.org/" Filename = Item_URL.replace(Local_URL, "") Filename = Filename.replace(".html/", "") Filename = Filename.replace(".html", "") Filename = Filename.replace("/", "-") Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Craigslist_Response, Filename, The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Plugin_Name, Local_Domain, "Search Result", Task_ID, Plugin_Name.lower()) Output_Connections.Output([Output_file], Item_URL, General.Get_Title(Item_URL), Plugin_Name.lower()) Data_to_Cache.append(Item_URL) else: logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") Current_Step += 1 if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w") except Exception as e: logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: # Query can be Title or ISBN Main_URL = f"http://{self.Domain}/search.php?req={Query}&lg_topic=libgen&open=0&view=simple&res=100&phrase=1&column=def" Lib_Gen_Response = Common.Request_Handler(Main_URL) Main_File = General.Main_File_Create(Directory, self.Plugin_Name, Lib_Gen_Response, Query, self.The_File_Extension) Lib_Gen_Regex = Common.Regex_Handler(Lib_Gen_Response, Custom_Regex=r"book\/index\.php\?md5=[A-Fa-f0-9]{32}", Findall=True) if Lib_Gen_Regex: Current_Step = 0 for Regex in Lib_Gen_Regex: Item_URL = f"http://{self.Domain}/{Regex}" Title = General.Get_Title(Item_URL).replace("Genesis:", "Genesis |") Lib_Item_Responses = Common.Request_Handler(Item_URL, Filter=True, Host=f"http://{self.Domain}") Lib_Item_Response = Lib_Item_Responses["Filtered"] if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(self.Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, Lib_Item_Response, Regex, self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Concat_Plugin_Name) Output_Connections.Output([Main_File, Output_file], Item_URL, Title, self.Concat_Plugin_Name) Data_to_Cache.append(Item_URL) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Current_Step += 1 else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match regular expression.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Location = self.Load_Configuration() Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Main_URL = f"https://www.{self.Domain}/en-{Location}/search?q={Query}" Win_Store_Response = Common.Request_Handler(Main_URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True) Main_File = General.Main_File_Create(Directory, self.Plugin_Name, Win_Store_Response, Query, self.The_File_Extension) Win_Store_Regex = Common.Regex_Handler(Win_Store_Response, Custom_Regex=r"\/en\-au\/p\/([\w\-]+)\/([\w\d]+)", Findall=True) Output_Connections = General.Connections(Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Concat_Plugin_Name) if Win_Store_Regex: Current_Step = 0 for Regex_Group_1, Regex_Group_2 in Win_Store_Regex: Item_URL = f"https://www.microsoft.com/en-au/p/{Regex_Group_1}/{Regex_Group_2}" Win_Store_Responses = Common.Request_Handler(Item_URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Win_Store_Response = Win_Store_Responses["Filtered"] Title = "Windows Store | " + General.Get_Title(Item_URL) if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(self.Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, Win_Store_Response, Regex_Group_1, self.The_File_Extension) if Output_file: Output_Connections.Output([Main_File, Output_file], Item_URL, Title, self.Concat_Plugin_Name) Data_to_Cache.append(Item_URL) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Current_Step += 1 else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match regular expression.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID): Data_to_Cache = [] Cached_Data = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: BSB_Search_URL = "https://www.bsbnumbers.com/" + Query + ".html" Response = requests.get(BSB_Search_URL).text Error_Regex = re.search(r"Correct\sthe\sfollowing\serrors", Response) if not Error_Regex: if BSB_Search_URL not in Cached_Data and BSB_Search_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Response, Query, The_File_Extension) if Output_file: General.Connections(Output_file, Query, Plugin_Name, BSB_Search_URL, "bsbnumbers.com", "Data Leakage", Task_ID, General.Get_Title(BSB_Search_URL), Plugin_Name.lower()) Data_to_Cache.append(BSB_Search_URL) else: logging.warning(General.Date() + " Query returned error, probably does not exist.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: Response = Common.Request_Handler('https://tpbc.herokuapp.com/search/' + Query.replace(" ", "+") + '/?sort=seeds_desc') JSON_Object = Common.JSON_Handler(Response) Response = JSON_Object.To_JSON_Loads() JSON_Response = JSON_Object.Dump_JSON() Output_file = General.Main_File_Create(Directory, Plugin_Name, JSON_Response, Query, The_File_Extension) if Output_file: Current_Step = 0 Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Torrent", Task_ID, Plugin_Name.lower()) for Search_Result in Response: Result_Title = Search_Result["title"] Result_URL = Search_Result["magnet"] if Result_URL not in Cached_Data and Result_URL not in Data_to_Cache and Current_Step < int(Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, JSON_Response, Result_Title, The_File_Extension) if Output_file: Output_Connections.Output([Output_file], Result_URL, General.Get_Title(Result_URL), Plugin_Name.lower()) Data_to_Cache.append(Result_URL) else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") Current_Step += 1 Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs): try: Data_to_Cache = [] Results = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Reddit_Details = Load_Configuration() Cached_Data = General.Get_Cache(Directory, Plugin_Name) Limit = General.Get_Limit(kwargs) Query_List = General.Convert_to_List(Query_List) for Query in Query_List: try: Reddit_Connection = praw.Reddit( client_id=Reddit_Details[0], client_secret=Reddit_Details[1], user_agent=Reddit_Details[2], username=Reddit_Details[3], password=Reddit_Details[4]) All_Subreddits = Reddit_Connection.subreddit(Reddit_Details[5]) for Subreddit in All_Subreddits.search( Query, limit=Limit ): # Limit, subreddit and search to be controlled by the web app. Results.append(Subreddit.url) except: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to get results. Are you connected to the internet?" ) Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Forum", Task_ID, Plugin_Name.lower()) for Result in Results: if Result not in Cached_Data and Result not in Data_to_Cache: try: Reddit_Regex = re.search( "https\:\/\/www\.reddit\.com\/r\/(\w+)\/comments\/(\w+)\/([\w\d]+)\/", Result[0]) if Reddit_Regex: Reddit_Responses = General.Request_Handler( Result, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}") Reddit_Response = Reddit_Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Reddit_Response, Reddit_Regex.group(3), The_File_Extension) if Output_file: Output_Connections.Output( [Output_file], Result, General.Get_Title(Result[0]), Plugin_Name.lower()) Data_to_Cache.append(Result[0]) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) except: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create file." ) General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name) except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs): Data_to_Cache = [] Cached_Data = [] Results = [] if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Reddit_Details = Load_Configuration() Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: try: Reddit_Connection = praw.Reddit(client_id=Reddit_Details[0], \ client_secret=Reddit_Details[1], \ user_agent=Reddit_Details[2], \ username=Reddit_Details[3], \ password=Reddit_Details[4]) All_Subreddits = Reddit_Connection.subreddit(Reddit_Details[5]) for Subreddit in All_Subreddits.search(Query, limit=Limit): # Limit, subreddit and search to be controlled by the web app. Current_Result = [] Current_Result.append(Subreddit.url) Current_Result.append(Subreddit.selftext) Results.append(Current_Result) except: logging.warning(General.Date() + " Failed to get results. Are you connected to the internet?") for Result in Results: if Result[0] not in Cached_Data and Result[0] not in Data_to_Cache: try: Reddit_Regex = re.search("https\:\/\/www\.reddit\.com\/r\/(\w+)\/comments\/(\w+)\/([\w\d]+)\/", Result[0]) if Reddit_Regex: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Result[1], Reddit_Regex.group(3), The_File_Extension) if Output_file: General.Connections(Output_file, Query, Plugin_Name, Result[0], "reddit.com", "Data Leakage", Task_ID, General.Get_Title(Result[0]), Plugin_Name.lower()) except: logging.warning(General.Date() + " Failed to create file.") Data_to_Cache.append(Result[0]) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Transaction_Search(Query_List, Task_ID, Type, **kwargs): try: Local_Plugin_Name = Plugin_Name + "-Transaction-Search" Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Local_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(kwargs) for Query in Query_List: if Type != "monero": if Type == "btc" or Type == "bch": Query_Regex = re.search(r"[\d\w]{64}", Query) elif Type == "eth": Query_Regex = re.search(r"(0x[\d\w]{64})", Query) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Invalid type provided." ) if Query_Regex: Main_URL = f"https://www.{Domain}/{Type}/tx/{Query}" Main_Response = General.Request_Handler(Main_URL) if Type == "btc": Address_Regex = re.findall( r"\/btc\/address\/([\d\w]{26,34})", Main_Response) elif Type == "bch": Address_Regex = re.findall(r"([\d\w]{42})", Main_Response) elif Type == "eth": Address_Regex = re.findall(r"(0x[\w\d]{40})", Main_Response) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Invalid type provided." ) if Address_Regex: Current_Step = 0 Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Blockchain Address", Task_ID, Plugin_Name.lower()) for Transaction in Address_Regex: Query_URL = f"https://www.{Domain}/{Type}/address/{Transaction}" if Query_URL not in Cached_Data and Query_URL not in Data_to_Cache and Current_Step < int( Limit): Transaction_Responses = General.Request_Handler( Query_URL, Filter=True, Host=f"https://www.{Domain}") Transaction_Response = Transaction_Responses[ "Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Transaction_Response, Transaction, The_File_Extension) if Output_file: Output_Connections.Output( [Output_file], Query_URL, General.Get_Title(Query_URL), Plugin_Name.lower()) Data_to_Cache.append(Query_URL) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression." ) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression." ) else: Alt_Domain = "localmonero.co" Query_URL = f"https://{Alt_Domain}/blocks/search/{Query}" Transaction_Response = General.Request_Handler(Query_URL) if "Whoops, looks like something went wrong." not in Transaction_Response and Query_URL not in Cached_Data and Query_URL not in Data_to_Cache: Transaction_Responses = General.Request_Handler( Query_URL, Filter=True, Host=f"https://{Alt_Domain}") Transaction_Response = Transaction_Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Transaction_Response, Query, The_File_Extension) if Output_file: Output_Connections = General.Connections( Query, Local_Plugin_Name, Alt_Domain, "Blockchain Transaction", Task_ID, Plugin_Name.lower()) Output_Connections.Output( [Output_file], Query_URL, General.Get_Title_Requests_Module(Query_URL), Plugin_Name.lower()) Data_to_Cache.append(Query_URL) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type_of_Query, **kwargs): Data_to_Cache = [] Cached_Data = [] try: if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) try: pyhibp.set_api_key(key=Load_Configuration()) except: logging.warning(General.Date() + " Failed to set API key, make sure it is set in the configuration file.") Query_List = General.Convert_to_List(Query_List) if Type_of_Query == "email": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pyhibp.get_pastes(email_address=Query) logging.info(Query_Response) if Query_Response: Domain = Query_Response[0]["Source"] ID = Query_Response[0]["Id"] Link = "https://www." + Domain + ".com/" + ID JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True) if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, JSON_Query_Response, "email", The_File_Extension) if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Link) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") elif Type_of_Query == "breach": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pyhibp.get_single_breach(breach_name=Query) if Query_Response: Domain = Query_Response["Domain"] Link = "https://www." + Domain + ".com/" JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True) if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "breach", The_File_Extension) if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Link) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") elif Type_of_Query == "password": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pw.is_password_breached(password=Query) logging.info(Query_Response) if Query_Response: Link = "https://haveibeenpwned.com/Passwords?" + Query if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, str(Query_Response), "password", ".txt") if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Link) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") elif Type_of_Query == "account": Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] for Query in Query_List: Query_Response = pyhibp.get_account_breaches(account=Query, truncate_response=True) if Query_Response: Current_Step = 0 for Response in Query_Response: Current_Response = pyhibp.get_single_breach(breach_name=Response['Name']) JSON_Query_Response = json.dumps(Current_Response, indent=4, sort_keys=True) Link = "https://" + Current_Response['Domain'] if Current_Response['Domain'] not in Cached_Data and Current_Response['Domain'] not in Data_to_Cache and Current_Step < int(Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "account", The_File_Extension) if Output_file: General.Connections(Output_file, Query, Local_Plugin_Name, Link, Current_Response['Domain'], "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower()) Data_to_Cache.append(Current_Response['Domain']) Current_Step += 1 if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w") else: logging.warning(General.Date() + " Invalid type provided.") except: logging.warning(General.Date() + " Execution error.")
def Search(Query_List, Task_ID, **kwargs): Data_to_Cache = [] Cached_Data = [] if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Craigslist_Location = Load_Configuration() Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: Main_URL = "https://" + Craigslist_Location.lower( ) + ".craigslist.org/search/sss?format=rss&query=" + Query Craigslist_Response = feedparser.parse(Main_URL) Craigslist_Items = Craigslist_Response["items"] Current_Step = 0 for Item in Craigslist_Items: Item_URL = Item["link"] if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int( Limit): Craigslist_Response = requests.get(Item_URL).text Local_URL = "https://" + Craigslist_Location.lower( ) + ".craigslist.org/" Local_Domain = Craigslist_Location.lower() + ".craigslist.org/" Filename = Item_URL.replace(Local_URL, "") Filename = Filename.replace(".html/", "") Filename = Filename.replace(".html", "") Filename = Filename.replace("/", "-") Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Craigslist_Response, Filename, The_File_Extension) if Output_file: General.Connections(Output_file, Query, Plugin_Name, Item_URL, Local_Domain, "Data Leakage", Task_ID, General.Get_Title(Item_URL), Plugin_Name.lower()) Data_to_Cache.append(Item_URL) Current_Step += 1 if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs): Data_to_Cache = [] Cached_Data = [] if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Ebay_API_Key = Load_Configuration() Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: try: API_Request = Connection(appid=Ebay_API_Key, config_file=None) API_Response = API_Request.execute('findItemsAdvanced', {'keywords': Query}) JSON_Output_Response = json.dumps(API_Response.dict(), indent=4, sort_keys=True) JSON_Response = json.dumps(API_Response.dict()) JSON_Response = json.loads(JSON_Response) General.Main_File_Create(Directory, Plugin_Name, JSON_Output_Response, Query, ".json") if JSON_Response["ack"] == "Success": Current_Step = 0 for JSON_Line in JSON_Response['searchResult']['item']: Ebay_Item_URL = JSON_Line['viewItemURL'] if Ebay_Item_URL not in Cached_Data and Ebay_Item_URL not in Data_to_Cache and Current_Step < int( Limit): Ebay_Item_Regex = re.search( r"http\:\/\/www\.ebay\.com\/itm\/([\w\d\-]+)\-\/\d+", Ebay_Item_URL) headers = { 'Content-Type': 'application/json', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0', 'Accept': 'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5' } Ebay_Item_Response = requests.get(Ebay_Item_URL, headers=headers).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Ebay_Item_Response, Ebay_Item_Regex.group(1), The_File_Extension) if Output_file: General.Connections( Output_file, Query, Plugin_Name, Ebay_Item_URL, "ebay.com", "Data Leakage", Task_ID, General.Get_Title(Ebay_Item_URL), Plugin_Name.lower()) Data_to_Cache.append(Ebay_Item_URL) Current_Step += 1 else: logging.warning(General.Date() + " No results found.") except: logging.info(General.Date() + " Failed to make API call.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs): Data_to_Cache = [] Cached_Data = [] if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 else: Limit = 10 Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) try: Flickr_Details = Load_Configuration() flickr_api.set_keys(api_key=Flickr_Details[0], api_secret=Flickr_Details[1]) except: logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to establish API identity.") for Query in Query_List: Email_Regex = re.search(r"[^@]+@[^\.]+\..+", Query) if Email_Regex: try: User = flickr_api.Person.findByEmail(Query) Photos = User.getPhotos() General.Main_File_Create(Directory, Plugin_Name, Photos, Query, ".txt") Output_Connections = General.Connections( Query, Plugin_Name, "flickr.com", "Data Leakage", Task_ID, Plugin_Name.lower()) Current_Step = 0 for Photo in Photos: Photo_URL = "https://www.flickr.com/photos/" + Query + "/" + Photo[ "id"] if Photo_URL not in Cached_Data and Photo_URL not in Data_to_Cache and Current_Step < int( Limit): headers = { 'Content-Type': 'application/json', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0', 'Accept': 'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5' } Photo_Response = requests.get(Photo_URL, headers=headers).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Photo_Response, Photo, The_File_Extension) if Output_file: Output_Connections.Output( Output_file, Photo_URL, General.Get_Title(Photo_URL)) Data_to_Cache.append(Photo_URL) Current_Step += 1 except: logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to make API call.") else: try: User = flickr_api.Person.findByUserName(Query) Photos = User.getPhotos() General.Main_File_Create(Directory, Plugin_Name, Photos, Query, ".txt") Output_Connections = General.Connections( Query, Plugin_Name, "flickr.com", "Data Leakage", Task_ID, Plugin_Name.lower()) Current_Step = 0 for Photo in Photos: Photo_URL = "https://www.flickr.com/photos/" + Query + "/" + Photo[ "id"] if Photo_URL not in Cached_Data and Photo_URL not in Data_to_Cache and Current_Step < int( Limit): headers = { 'Content-Type': 'application/json', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0', 'Accept': 'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5' } Photo_Response = requests.get(Photo_URL, headers=headers).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Photo_Response, str(Photo['id']), The_File_Extension) if Output_file: Output_Connections.Output( Output_file, Photo_URL, General.Get_Title(Photo_URL)) Data_to_Cache.append(Photo_URL) Current_Step += 1 except: logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to make API call.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: if Type == "User": from instagramy import InstagramUser Local_Plugin_Name = Plugin_Name + "-" + Type CSE_Response = InstagramUser(Query) JSON_Object = Common.JSON_Handler(vars(CSE_Response)) CSE_JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, The_File_Extensions["Main"]) if not CSE_Response.is_private: Posts = CSE_Response.posts Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Social Media - Person", Task_ID, Local_Plugin_Name.lower()) Current_Step = 0 for Post in Posts: URL = Post['post_url'] Shortcode = URL.replace(f"https://www.{Domain}/p/", "").replace("/", "") Title = "IG | " + General.Get_Title(URL, Requests=True) if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}") Response = Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Response, Shortcode, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.info( f"{Common.Date()} - {__name__.strip('plugins.')} - The provided user's profile is private and cannot be scraped." ) elif Type == "Tag": from instagramy import InstagramHashTag Local_Plugin_Name = Plugin_Name + "-" + Type CSE_Response = InstagramHashTag(Query) JSON_Object = Common.JSON_Handler(vars(CSE_Response)) CSE_JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, The_File_Extensions["Main"]) Posts = vars( CSE_Response)['tag_data']['edge_hashtag_to_media']['edges'] Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Social Media - Person", Task_ID, Local_Plugin_Name.lower()) Current_Step = 0 for Post in Posts: Shortcode = Post['node']['shortcode'] URL = f"https://www.{Domain}/p/{Shortcode}/" Title = "IG | " + General.Get_Title(URL, Requests=True) if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}") Response = Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Response, Shortcode, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 elif Type == "Post": from instagramy import InstagramPost Local_Plugin_Name = Plugin_Name + "-" + Type CSE_Response = InstagramPost(Query) JSON_Object = Common.JSON_Handler(vars(CSE_Response)) CSE_JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Social Media - Place", Task_ID, Local_Plugin_Name.lower()) URL = CSE_Response.url Shortcode = URL.replace(f"https://www.{Domain}/p/", "").replace("/", "") Title = "IG | " + General.Get_Title(URL, Requests=True) if URL not in Cached_Data and URL not in Data_to_Cache: Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}") Response = Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Response, Shortcode, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid type provided." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID): Data_to_Cache = [] Cached_Data = [] Configuration_Details = Load_Configuration() Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: if Configuration_Details[1].lower() == "true": Request = 'https://api.certspotter.com/v1/issuances?domain=' + Query + '&include_subdomains=true&expand=dns_names&expand=issuer&expand=cert' Response = requests.get(Request, auth=(Configuration_Details[0], '')).text else: Request = 'https://api.certspotter.com/v1/issuances?domain=' + Query + '&expand=dns_names&expand=issuer&expand=cert' Response = requests.get(Request, auth=(Configuration_Details[0], '')).text JSON_Response = json.loads(Response) if 'exists' not in JSON_Response: if JSON_Response: if Request not in Cached_Data and Request not in Data_to_Cache: try: SSLMate_Regex = re.search( "([\w\d]+)\.[\w]{2,3}(\.[\w]{2,3})?(\.[\w]{2,3})?", Query) if SSLMate_Regex: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, json.dumps(JSON_Response, indent=4, sort_keys=True), SSLMate_Regex.group(1), The_File_Extension) if Output_file: Output_Connections = General.Connections( Query, Plugin_Name, "sslmate.com", "Domain Spoof", Task_ID, Plugin_Name.lower()) Output_Connections.Output( Output_file, Request, General.Get_Title(Request)) else: logging.warning( General.Date() + " - " + __name__.strip('plugins.') + " - Failed to match regular expression.") except: logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to create file.") Data_to_Cache.append(Request) else: logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - No response.") else: logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Query does not exist.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(kwargs) try: Flickr_Details = Load_Configuration() flickr_api.set_keys(api_key=Flickr_Details[0], api_secret=Flickr_Details[1]) except: logging.info( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to establish API identity." ) for Query in Query_List: Email_Regex = re.search(r"[^@]+@[^\.]+\..+", Query) if Email_Regex: try: User = flickr_api.Person.findByEmail(Query) Photos = User.getPhotos() if Photos: Main_File = General.Main_File_Create( Directory, Plugin_Name, Convert_to_JSON(Photos), Query, The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Plugin_Name, "flickr.com", "Social Media - Media", Task_ID, Plugin_Name.lower()) Current_Step = 0 for Photo in Photos: Photo_URL = f"https://www.flickr.com/photos/{Query}/{Photo['id']}" if Photo_URL not in Cached_Data and Photo_URL not in Data_to_Cache and Current_Step < int( Limit): Photo_Response = requests.get( Photo_URL, headers=headers).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Photo_Response, Photo, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Photo_URL, General.Get_Title(Photo_URL), Plugin_Name.lower()) Data_to_Cache.append(Photo_URL) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - No photos found." ) except: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to make API call." ) else: try: User = flickr_api.Person.findByUserName(Query) Photos = User.getPhotos() if Photos: Main_File = General.Main_File_Create( Directory, Plugin_Name, Convert_to_JSON(Photos), Query, The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Plugin_Name, "flickr.com", "Data Leakage", Task_ID, Plugin_Name.lower()) Current_Step = 0 for Photo in Photos: Photo_URL = f"https://www.flickr.com/photos/{Query}/{Photo['id']}" if Photo_URL not in Cached_Data and Photo_URL not in Data_to_Cache and Current_Step < int( Limit): Photo_Response = requests.get( Photo_URL, headers=headers).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Photo_Response, str(Photo['id']), The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Photo_URL, General.Get_Title(Photo_URL), Plugin_Name.lower()) Data_to_Cache.append(Photo_URL) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - No photos found." ) except: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to make API call." ) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w") except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type, **kwargs): Data_to_Cache = [] Cached_Data = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: try: if Type == "ABN": Main_URL = 'https://abr.business.gov.au/ABN/View?id=' + Query Response = requests.get(Main_URL).text try: if 'Error searching ABN Lookup' not in Response: Query = str(int(Query)) if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Response, General.Get_Title(Main_URL), The_File_Extension) if Output_file: General.Connections( Output_file, Query, Plugin_Name, Main_URL, "abr.business.gov.au", "Data Leakage", Task_ID, General.Get_Title(Main_URL), Plugin_Name) Data_to_Cache.append(Main_URL) except: logging.warning(General.Date() + " Invalid query provided for ABN Search.") elif Type == "ACN": Main_URL = 'https://abr.business.gov.au/Search/Run' Data = { 'SearchParameters.SearchText': Query, 'SearchParameters.AllNames': 'true', 'ctl00%24ContentPagePlaceholder%24SearchBox%24MainSearchButton': 'Search' } Response = requests.post(Main_URL, data=Data).text if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 try: ACN_Regex = re.search(r".*[a-zA-Z].*", Query) if ACN_Regex: General.Main_File_Create(Directory, Plugin_Name, Response, Query, The_File_Extension) Current_Step = 0 ABNs_Regex = re.findall( r"\<input\sid\=\"Results\_NameItems\_\d+\_\_Compressed\"\sname\=\"Results\.NameItems\[\d+\]\.Compressed\"\stype\=\"hidden\"\svalue\=\"(\d{11})\,\d{2}\s\d{3}\s\d{3}\s\d{3}\,0000000001\,Active\,active\,([\d\w\s\&\-\_\.]+)\,Current\,", Response) if ABNs_Regex: for ABN_URL, ACN in ABNs_Regex: Full_ABN_URL = 'https://abr.business.gov.au/ABN/View?abn=' + ABN_URL if Full_ABN_URL not in Cached_Data and Full_ABN_URL not in Data_to_Cache and Current_Step < int( Limit): ACN = ACN.rstrip() Current_Response = requests.get( Full_ABN_URL).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, str(Current_Response), ACN.replace(' ', '-'), The_File_Extension) if Output_file: General.Connections( Output_file, Query, Plugin_Name, Full_ABN_URL, "abr.business.gov.au", "Data Leakage", Task_ID, General.Get_Title(Full_ABN_URL), Plugin_Name) Data_to_Cache.append(Full_ABN_URL) Current_Step += 1 else: logging.warning( General.Date() + " Response did not match regular expression.") else: logging.warning( General.Date() + " Query did not match regular expression.") except: logging.warning(General.Date() + " Invalid query provided for ACN Search.") else: logging.warning(General.Date() + " Invalid request type.") except: logging.warning(General.Date() + " Failed to make request.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, **kwargs): Data_to_Cache = [] Cached_Data = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: try: if Type == "NZBN": Main_URL = 'https://app.companiesoffice.govt.nz/companies/app/ui/pages/companies/search?q=' + Query + '&entityTypes=ALL&entityStatusGroups=ALL&incorpFrom=&incorpTo=&addressTypes=ALL&addressKeyword=&start=0&limit=1&sf=&sd=&advancedPanel=true&mode=advanced#results' Response = requests.get(Main_URL).text try: if 'An error has occurred and the requested action cannot be performed.' not in Response: Query = str(int(Query)) if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Response, General.Get_Title(Main_URL), The_File_Extension) if Output_file: General.Connections(Output_file, Query, Plugin_Name, Main_URL, "app.companiesoffice.govt.nz", "Data Leakage", Task_ID, General.Get_Title(Main_URL), Plugin_Name) Data_to_Cache.append(Main_URL) except: logging.warning(General.Date() + " Invalid query provided for NZBN Search.") elif Type == "NZCN": if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 try: Main_URL = 'https://app.companiesoffice.govt.nz/companies/app/ui/pages/companies/search?q=' + urllib.parse.quote( Query) + '&entityTypes=ALL&entityStatusGroups=ALL&incorpFrom=&incorpTo=&addressTypes=ALL&addressKeyword=&start=0&limit=' + str( Limit) + '&sf=&sd=&advancedPanel=true&mode=advanced#results' Response = requests.get(Main_URL).text NZCN_Regex = re.search(r".*[a-zA-Z].*", Query) if NZCN_Regex: General.Main_File_Create(Directory, Plugin_Name, Response, Query, The_File_Extension) NZBNs_Regex = re.findall(r"\<span\sclass\=\"entityName\"\>([\w\d\s\-\_\&\|\!\@\#\$\%\^\*\(\)\.\,]+)\<\/span\>\s<span\sclass\=\"entityInfo\"\>\((\d{6})\)\s\(NZBN\:\s(\d{13})\)", Response) if NZBNs_Regex: for NZCN, NZ_ID, NZBN_URL in NZBNs_Regex: print(NZBN_URL, NZ_ID, NZCN) Full_NZBN_URL = 'https://app.companiesoffice.govt.nz/companies/app/ui/pages/companies/' + NZ_ID + '?backurl=H4sIAAAAAAAAAEXLuwrCQBCF4bfZNtHESIpBbLQwhWBeYNgddSF7cWai5O2NGLH7zwenyHgjKWwKGaOfSwjZ3ncPaOt1W9bbsmqaamMoqtepnzIJ7Ltu2RdFHeXIacxf9tEmzgdOAZbuExh0jknk%2F17gRNMrsQMjiqxQmsEHr7Aycp3NfY5PjJbcGSMNoDySCckR%2FPwNLgXMiL4AAAA%3D' if Full_NZBN_URL not in Cached_Data and Full_NZBN_URL not in Data_to_Cache: Current_Response = requests.get(Full_NZBN_URL).text Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, str(Current_Response), NZCN.replace(' ', '-'), The_File_Extension) if Output_file: General.Connections(Output_file, Query, Plugin_Name, Full_NZBN_URL, "app.companiesoffice.govt.nz", "Data Leakage", Task_ID, General.Get_Title(Full_NZBN_URL), Plugin_Name) Data_to_Cache.append(Full_NZBN_URL) else: logging.warning(General.Date() + " Response did not match regular expression.") else: logging.warning(General.Date() + " Query did not match regular expression.") except: logging.warning(General.Date() + " Invalid query provided for NZCN Search.") else: logging.warning(General.Date() + " Invalid request type.") except: logging.warning(General.Date() + " Failed to make request.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, **kwargs): Data_to_Cache = [] Cached_Data = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: if Type == "CIK": Main_URL = 'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=' + Query + '&owner=exclude&count=40&hidefilings=0' Response = requests.get(Main_URL).text try: if 'No matching CIK.' not in Response: Query = str(int(Query)) if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Response, General.Get_Title(Main_URL), The_File_Extension) if Output_file: General.Connections(Output_file, Query, Plugin_Name, Main_URL, "sec.gov", "Data Leakage", Task_ID, General.Get_Title(Main_URL), Plugin_Name) Data_to_Cache.append(Main_URL) except: logging.info( str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + " Invalid query provided for CIK Search.") elif Type == "ACN": Main_URL = 'https://www.sec.gov/cgi-bin/browse-edgar?company=' + Query + '&owner=exclude&action=getcompany' Response = requests.get(Main_URL).text if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 try: ACN = re.search(r".*[a-zA-Z].*", Query) if ACN: General.Main_File_Create(Directory, Plugin_Name, Response, Query, The_File_Extension) Current_Step = 0 CIKs_Regex = re.findall( r"(\d{10})\<\/a\>\<\/td\>\s+\<td\sscope\=\"row\"\>(.*\S.*)\<\/td\>", Response) if CIKs_Regex: for CIK_URL, ACN in CIKs_Regex: Full_CIK_URL = 'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=' + CIK_URL + '&owner=exclude&count=40&hidefilings=0' if Full_CIK_URL not in Cached_Data and Full_CIK_URL not in Data_to_Cache and Current_Step < int( Limit): Current_Response = requests.get( Full_CIK_URL).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, str(Current_Response), ACN.replace(' ', '-'), The_File_Extension) if Output_file: General.Connections( Output_file, Query, Plugin_Name, Full_CIK_URL, "sec.gov", "Data Leakage", Task_ID, General.Get_Title(Full_CIK_URL), Plugin_Name) Data_to_Cache.append(Full_CIK_URL) Current_Step += 1 except: logging.info( str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + " Invalid query provided for ACN Search.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs): Data_to_Cache = [] Cached_Data = [] if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 else: Limit = 10 Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: # Query can be Title or ISBN Main_URL = "http://gen.lib.rus.ec/search.php?req=" + Query + "&lg_topic=libgen&open=0&view=simple&res=100&phrase=1&column=def" Lib_Gen_Response = requests.get(Main_URL).text General.Main_File_Create(Directory, Plugin_Name, Lib_Gen_Response, Query, The_File_Extension) Lib_Gen_Regex = re.findall("book\/index\.php\?md5=[A-Fa-f0-9]{32}", Lib_Gen_Response) if Lib_Gen_Regex: Current_Step = 0 for Regex in Lib_Gen_Regex: Item_URL = "http://gen.lib.rus.ec/" + Regex Lib_Item_Response = requests.get(Item_URL).text if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int( Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Lib_Item_Response, Regex, The_File_Extension) if Output_file: General.Connections(Output_file, Query, Plugin_Name, Item_URL, "gen.lib.rus.ec", "Data Leakage", Task_ID, General.Get_Title(Item_URL), Concat_Plugin_Name) Data_to_Cache.append(Item_URL) Current_Step += 1 else: logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to match regular expression.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Character_Switch(Query_List, Task_ID): Local_Plugin_Name = Plugin_Name + "-Character-Switch" Data_to_Cache = [] Cached_Data = [] Valid_Results = ["Domain,IP Address"] Valid_Hosts = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Local_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name) if not Cached_Data: Cached_Data = [] logging.info( str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + " Character Switching Selected.") Query_List = General.Convert_to_List(Query_List) for Query in Query_List: URL_Regex = re.search( r"(https?:\/\/(www\.)?)?([-a-zA-Z0-9@:%_\+~#=]{2,256})(\.[a-z]{2,3})(\.[a-z]{2,3})?(\.[a-z]{2,3})?", Query) if URL_Regex: URL_Prefix = URL_Regex.group(1) URL_Body = URL_Regex.group(3) if URL_Regex.group(5) and URL_Regex.group(6): URL_Extension = URL_Regex.group(4) + URL_Regex.group( 5) + URL_Regex.group(6) elif URL_Regex.group(5): URL_Extension = URL_Regex.group(4) + URL_Regex.group(5) else: URL_Extension = URL_Regex.group(4) else: logging.warning( str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + " Please provide valid URLs.") logging.info( str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + URL_Body) URL_List = list(URL_Body) Altered_URLs = Rotor.Search(URL_List, True, False, False, False, True, True, True) logging.info( str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + ", ".join(Altered_URLs)) for Altered_URL in Altered_URLs: if not Altered_URL == URL_Body: try: Query = Altered_URL + URL_Extension Web_Host = URL_Prefix.replace("s", "") + Query Response = socket.gethostbyname(Query) if Response: Cache = Query + ":" + Response if Cache not in Cached_Data and Cache not in Data_to_Cache: Valid_Results.append(Query + "," + Response) Data_to_Cache.append(Cache) Valid_Hosts.append(Web_Host) except Exception as e: logging.info( str(datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S')) + str(e)) logging.info( str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + Directory) URL_Domain = URL_Body + URL_Extension Output_File = General.Main_File_Create(Directory, Local_Plugin_Name, "\n".join(Valid_Results), URL_Body, The_File_Extension) if Output_File: for Host in Valid_Hosts: General.Connections(Output_File, Query, Local_Plugin_Name, Host, URL_Domain, "Domain Spoof", Task_ID, General.Get_Title(Host), Local_Plugin_Name.lower()) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs): Data_to_Cache = [] Cached_Data = [] if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: Tor_Pull_URL = Tor_General_URL + Query Tor_Scrape_URLs = General.Get_Latest_URLs(Tor_Pull_URL, Tor_Scrape_Regex_URL) if Tor_Scrape_URLs: Output_file = General.Main_File_Create(Directory, Tor_Plugin_Name.lower(), "\n".join(Tor_Scrape_URLs), Query, The_File_Extension) if Output_file: Current_Step = 0 for URL in Tor_Scrape_URLs: if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): General.Connections(Output_file, Query, Tor_Plugin_Name, URL, "ahmia.fl", "Domain Spoof", Task_ID, General.Get_Title(URL), Plugin_Name.lower()) Data_to_Cache.append(URL) Current_Step += 1 else: logging.info(General.Date() + " No Tor links scraped.") I2P_Pull_URL = I2P_General_URL + Query I2P_Scrape_URLs = General.Get_Latest_URLs(I2P_Pull_URL, I2P_Scrape_Regex_URL) if I2P_Scrape_URLs: Output_file = General.Main_File_Create(Directory, I2P_Plugin_Name.lower(), "\n".join(I2P_Scrape_URLs), Query, The_File_Extension) if Output_file: Current_Step = 0 for URL in I2P_Scrape_URLs: if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): General.Connections(Output_file, Query, I2P_Plugin_Name, URL, "ahmia.fl", "Domain Spoof", Task_ID, General.Get_Title(URL), Plugin_Name.lower()) Data_to_Cache.append(URL) Current_Step += 1 else: logging.info(General.Date() + " No I2P links scraped.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Location = Connectors.Load_Location_Configuration() Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: try: Request_Query = urllib.parse.quote(Query) Main_URL = f"http://{Domain}/search?term={Request_Query}&country={Location}&entity=software&limit={str(Limit)}" Response = Common.Request_Handler(Main_URL) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to make request, are you connected to the internet?" ) break JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Main_File = General.Main_File_Create(Directory, Plugin_Name, JSON_Object.Dump_JSON(), Query, The_File_Extensions["Main"]) if 'resultCount' in JSON_Response: if JSON_Response['resultCount'] > 0: Output_Connections = General.Connections( Query, Plugin_Name, Domain, "Application", Task_ID, Concat_Plugin_Name) for JSON_Object in JSON_Response['results']: JSON_Object_Responses = Common.Request_Handler( JSON_Object['artistViewUrl'], Filter=True, Host=f"https://{Domain}") JSON_Object_Response = JSON_Object_Responses[ "Filtered"] if JSON_Object[ 'artistViewUrl'] not in Cached_Data and JSON_Object[ 'artistViewUrl'] not in Data_to_Cache: Apple_Store_Regex = Common.Regex_Handler( JSON_Object['artistViewUrl'], Custom_Regex=r"https\:\/\/apps\.apple\.com\/" + rf"{Location}" + r"\/developer\/[\w\d\-]+\/(id[\d]{9,10})\?.+") if Apple_Store_Regex: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, JSON_Object_Response, Apple_Store_Regex.group(1), The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], JSON_Object['artistViewUrl'], General.Get_Title( JSON_Object['artistViewUrl']), Concat_Plugin_Name) Data_to_Cache.append( JSON_Object['artistViewUrl']) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid value provided, value not greater than 0." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid value." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) for Query in Query_List: try: if Type == "CBN": Main_API_URL = f'https://searchapi.mrasservice.ca/Search/api/v1/search?fq=keyword:%7B{Query}%7D+Status_State:Active&lang=en&queryaction=fieldquery&sortfield=Company_Name&sortorder=asc' Response = Common.Request_Handler(Main_API_URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Indented_JSON_Response = JSON_Object.Dump_JSON() Main_Output_File = General.Main_File_Create( Directory, Plugin_Name, Indented_JSON_Response, Query, The_File_Extensions["Main"]) try: if JSON_Response['count'] != 0: Query = str(int(Query)) Main_URL = f'https://{Domain}/search/results?search=%7B{Query}%7D&status=Active' Responses = Common.Request_Handler( Main_URL, Filter=True, Host=f"https://{Domain}") Response = Responses["Filtered"] if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Response, General.Get_Title(Main_URL), The_File_Extensions["Query"]) if Output_file: Output_Connections = General.Connections( Query, Plugin_Name, Domain.strip("beta."), "Company Details", Task_ID, Plugin_Name) Output_Connections.Output( [Main_Output_File, Output_file], Main_URL, f"Canadian Business Number {Query}", Concat_Plugin_Name) Data_to_Cache.append(Main_URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid query provided for CBN Search." ) elif Type == "CCN": Total_Results = 0 Iterator = "page=0" while Limit > Total_Results and Iterator is not None: Main_URL = 'https://searchapi.mrasservice.ca/Search/api/v1/search?fq=keyword:%7B' + urllib.parse.quote( Query ) + f'%7D+Status_State:Active&lang=en&queryaction=fieldquery&sortfield=Company_Name&sortorder=asc&{Iterator}' Response = Common.Request_Handler(Main_URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Total_Results += len(JSON_Response["docs"]) if "paging" in JSON_Response and "next" in JSON_Response.get( "paging"): Iterator = JSON_Response["paging"]["next"] else: Iterator = None Indented_JSON_Response = JSON_Object.Dump_JSON() Limit = General.Get_Limit(Limit) try: Main_File = General.Main_File_Create( Directory, Plugin_Name, Indented_JSON_Response, Query, The_File_Extensions["Main"]) Current_Step = 0 Output_Connections = General.Connections( Query, Plugin_Name, Domain.strip("beta."), "Company Details", Task_ID, Plugin_Name) for JSON_Item in JSON_Response['docs']: if JSON_Item.get('BN'): CCN = JSON_Item['Company_Name'] CBN = str(int(JSON_Item['BN'])) Full_CCN_URL = f'https://{Domain}/search/results?search=%7B{CBN}%7D&status=Active' if Full_CCN_URL not in Cached_Data and Full_CCN_URL not in Data_to_Cache and Current_Step < int( Limit): Current_Responses = Common.Request_Handler( Full_CCN_URL, Filter=True, Host=f"https://{Domain}") Current_Response = Current_Responses[ "Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, str(Current_Response), CCN.replace(' ', '-'), The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Full_CCN_URL, f"Canadian Business Number {CBN} for Query {Query}", Concat_Plugin_Name) Data_to_Cache.append(Full_CCN_URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Unable to retrieve business numbers from the JSON response." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid query provided for CCN Search." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid request type." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to make request." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) try: pyhibp.set_api_key(key=Load_Configuration()) except: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to set API key, make sure it is set in the configuration file.") if self.Type == "email": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_pastes(email_address=Query) logging.info(Query_Response) if Query_Response: Current_Domain = Query_Response[0]["Source"] ID = Query_Response[0]["Id"] Link = f"https://www.{Current_Domain}.com/{ID}" JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, JSON_Query_Response, "email", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_1, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "breach": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_single_breach(breach_name=Query) if Query_Response: Current_Domain = Query_Response["Domain"] Link = f"https://www.{Current_Domain}.com/" JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "breach", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_2, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "password": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pw.is_password_breached(password=Query) logging.info(Query_Response) if Query_Response: Link = f"https://{self.Domain}/Passwords?{Query}" if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, str(Query_Response), "password", ".txt") if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_2, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "account": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_account_breaches(account=Query, truncate_response=True) if Query_Response: Current_Step = 0 for Response in Query_Response: Current_Response = pyhibp.get_single_breach(breach_name=Response['Name']) JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() Link = "https://" + Current_Response['self.Domain'] if Current_Response['self.Domain'] not in Cached_Data and Current_Response['self.Domain'] not in Data_to_Cache and Current_Step < int(self.Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "account", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, Current_Response['self.Domain'], self.Result_Type_1, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Current_Response['self.Domain']) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Current_Step += 1 Cached_Data_Object.Write_Cache(Data_to_Cache) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid Type provided.") except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(kwargs) for Query in Query_List: URL_Query = urllib.parse.quote(Query) URL = f"https://api.duckduckgo.com/?q={URL_Query}&format=json" DDG_Response = requests.get(URL).text JSON_Response = json.loads(DDG_Response) JSON_Output_Response = json.dumps(JSON_Response, indent=4, sort_keys=True) Main_File = General.Main_File_Create(Directory, Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"]) Output_Connections = General.Connections(Query, Plugin_Name, "duckduckgo.com", "Search Result", Task_ID, Plugin_Name.lower()) if JSON_Response.get('RelatedTopics'): Current_Step = 0 for DDG_Item_Link in JSON_Response['RelatedTopics']: try: if 'FirstURL' in DDG_Item_Link: DDG_URL = DDG_Item_Link['FirstURL'] Title = General.Get_Title(DDG_URL) Title = f"DuckDuckGo | {Title}" if DDG_URL not in Cached_Data and DDG_URL not in Data_to_Cache and Current_Step < int( Limit): headers = { 'Content-Type': 'application/json', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0', 'Accept': 'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5' } DDG_Item_Response = requests.get( DDG_URL, headers=headers).text Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, DDG_Item_Response, DDG_URL, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], DDG_URL, Title, Plugin_Name.lower()) Data_to_Cache.append(DDG_URL) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: break elif 'Topics' in DDG_Item_Link: if type(DDG_Item_Link['Topics']) == list: JSON_Response['RelatedTopics'].extend( DDG_Item_Link['Topics']) except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}" ) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - No results found." ) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w") except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def General_Pull(Handle, Limit, Directory, API, Task_ID): Data_to_Cache = [] Cached_Data = [] JSON_Response = [] Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Latest_Tweets = API.user_timeline(screen_name=Handle, count=Limit) for Tweet in Latest_Tweets: Link = "" try: JSON_Response.append({ 'id': Tweet.id, 'text': Tweet.text, 'author_name': Tweet.user.screen_name, 'url': Tweet.entities['urls'][0]["expanded_url"] }) Link = Tweet.entities['urls'][0]["expanded_url"] except: JSON_Response.append({ 'id': Tweet.id, 'text': Tweet.text, 'author_name': Tweet.user.screen_name }) JSON_Output = json.dumps(JSON_Response, indent=4, sort_keys=True) for JSON_Item in JSON_Response: if 'text' in JSON_Item and 'url' in JSON_Item: Link = JSON_Item['url'] if Link not in Cached_Data and Link not in Data_to_Cache: logging.info(General.Date() + " " + Link) Item_Response = requests.get(Link).text Output_file = General.Create_Query_Results_Output_File(Directory, Handle, Plugin_Name, Item_Response, str(JSON_Item['id']), ".html") if Output_file: General.Connections(Output_file, Handle, Plugin_Name, Link, "twitter.com", "Data Leakage", Task_ID, General.Get_Title(Link), Plugin_Name.lower()) else: logging.warning(General.Date() + " Output file not returned.") else: logging.warning(General.Date() + " Insufficient parameters provided.") Data_to_Cache.append(Link) General.Main_File_Create(Directory, Plugin_Name, JSON_Output, Handle, ".json") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) try: File_Dir = os.path.dirname(os.path.realpath('__file__')) Configuration_File = os.path.join( File_Dir, 'plugins/common/config/RSS_Feeds.txt') Current_File = open( Configuration_File, "r" ) # Open the provided file and retrieve each client to test. URLs = Current_File.read().splitlines() Current_File.close() except: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Please provide a valid file, failed to open the file which contains the data to search for." ) Cached_Data = General.Get_Cache(Directory, Plugin_Name) Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(kwargs) for Query in Query_List: for URL in URLs: # URLs to be controlled by the web app. RSS = feedparser.parse(URL) Current_Step = 0 for Feed in RSS.entries: if Query in Feed.description: Dump_Types = General.Data_Type_Discovery( Feed.description) File_Link = Feed.link.replace("https://", "") File_Link = File_Link.replace("http://", "") File_Link = File_Link.replace("www.", "") File_Link = File_Link.replace("/", "-") Domain = URL.replace("https://", "") Domain = Domain.replace("http://", "") Domain = Domain.replace("www.", "") if Feed.link not in Cached_Data and Feed.link not in Data_to_Cache and Current_Step < int( Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Feed.description, File_Link, The_File_Extension) Title = "RSS Feed | " + General.Get_Title( Feed.link) if Output_file: Output_Connections = General.Connections( Query, Plugin_Name, Domain, "News Report", Task_ID, Plugin_Name.lower()) Output_Connections.Output( [Output_file], Feed.link, Title, Plugin_Name.lower(), Dump_Types=Dump_Types) Data_to_Cache.append(Feed.link) else: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.info( f"{General.Date()} - {__name__.strip('plugins.')} - Query not found." ) if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w") except Exception as e: logging.warning( f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: URL_Body = f'https://{self.Domain}' Main_URL = URL_Body + '/' + Query.lower().replace(' ', '-') Responses = Common.Request_Handler( Main_URL, Filter=True, Host=f"https://www.{self.Domain}") Response = Responses["Regular"] Filtered_Response = Responses["Filtered"] Main_File = General.Main_File_Create(Directory, self.Plugin_Name, Filtered_Response, Query, self.The_File_Extension) Regex = Common.Regex_Handler( Response, Custom_Regex= r"\<tr\>\s+\<td\sclass\=\"name\"\>\s+\<a\shref\=\"([\/\d\w\-\+\?\.]+)\"\>([\/\d\w\-\+\?\.\(\)\s\,\;\:\~\`\!\@\#\$\%\^\&\*\[\]\{\}]+)\<\/a\>\s+\<\/td\>", Findall=True) if Regex: Current_Step = 0 Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Concat_Plugin_Name) for URL, Title in Regex: Item_URL = URL_Body + URL Current_Response = Common.Request_Handler(Item_URL) Current_Item_Regex = Common.Regex_Handler( Current_Response, Custom_Regex= r"\<button\sclass\=\"btn\sbtn\-primary\spassword\"\s+data\-data\=\"([\-\d\w\?\/]+)\"\s+data\-toggle\=\"modal\"\s+data\-target\=\"\#modal\"\s+\>show\sme\!\<\/button\>" ) if Current_Item_Regex: try: Detailed_Item_URL = URL_Body + Current_Item_Regex.group( 1) Detailed_Responses = Common.Request_Handler( Item_URL, Filter=True, Host=f"https://www.{self.Domain}") Detailed_Response = Detailed_Responses[ "Regular"] Output_Dict = Common.JSON_Handler( Detailed_Response).Is_JSON() if JSON_Response: Output_Response = "<head><title>" + JSON_Response[ "title"] + "</title></head>\n" Output_Response = Output_Response + JSON_Response[ "data"] else: Output_Response = Detailed_Responses[ "Filtered"] if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int( self.Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Output_Response, Title, self.The_File_Extension) if Output_file: Output_Connections.Output( [Main_File, Output_file], Item_URL, General.Get_Title(Item_URL), self.Concat_Plugin_Name) Data_to_Cache.append(Item_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) Current_Step += 1 except: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to generate output, may have a blank detailed response." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match regular expression for current result." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match regular expression for provided query." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")