def CSV_to_JSON(Query, CSV_Data): try: if type(CSV_Data) == list: JSON_Data = {Query: []} for CSV_Line in CSV_Data: if CSV_Line != CSV_Data[0]: Split_CSV_Line = CSV_Line.split(",") JSON_Data[Query].append({ "Domain": Split_CSV_Line[0], "IP Address": Split_CSV_Line[1] }) Indented_Registration_Response = Common.JSON_Handler( JSON_Data).Dump_JSON() return Indented_Registration_Response else: return None except: logging.warning( f"{Common.Date()} - General Library - Failed to convert provided CSV data to JSON." )
def Search(Query_List, Task_ID): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) for Query in Query_List: if Common.Regex_Handler(Query, Type="IP"): API_Key = Load_Configuration() Search_Response = Common.Request_Handler( f"http://api.{Domain}/{Query}?access_key={API_Key}") JSON_Object = Common.JSON_Handler(Search_Response) JSON_Response = JSON_Object.To_JSON_Loads() JSON_Output_Response = JSON_Object.Dump_JSON() Output_Connections = General.Connections( Query, Plugin_Name, Domain, "IP Address Information", Task_ID, Plugin_Name.lower()) if Query not in Cached_Data and Query not in Data_to_Cache: Result_URL = f"https://{Domain}/?{Query}" Title = f"IP Stack | {Query}" Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, JSON_Output_Response, Title, The_File_Extensions["Main"]) HTML_Output_File_Data = General.JSONDict_to_HTML( JSON_Response, JSON_Output_Response, f"IPStack Query {Query}") HTML_Output_File = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, HTML_Output_File_Data, Title.replace(" ", "-"), The_File_Extensions["Main_Converted"]) if Output_file: Output_Connections.Output([Output_file], Result_URL, Title, Plugin_Name.lower()) Data_to_Cache.append(Result_URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: if Common.Regex_Handler(Query, Type="Email"): Link = f"https://{self.Domain}/home/verify-as-guest/{Query}" JSON_Response = Common.Request_Handler(Link) JSON_Object = Common.JSON_Handler(JSON_Response) if JSON_Object.Is_JSON(): JSON_Response = JSON_Object.To_JSON_Loads() JSON_Output_Response = JSON_Object.Dump_JSON() Table_JSON = {} for Key, Value in JSON_Response.items(): if Key != "response": Table_JSON[Key] = Value else: for Det_Key, Det_Val in JSON_Response["response"].items(): Table_JSON[Det_Key] = Det_Val Filter_JSON = [Table_JSON] Output_Connections = General.Connections(Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Concat_Plugin_Name) if Query not in Cached_Data and Query not in Data_to_Cache: Title = f"Email Verification | {Query}" Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Concat_Plugin_Name, JSON_Output_Response, Title, self.The_File_Extensions["Main"]) HTML_Output_File_Data = General.JSONDict_to_HTML(Filter_JSON, JSON_Output_Response, f"Email Verification Query {Query}") HTML_Output_File = General.Create_Query_Results_Output_File(Directory, Query, self.Concat_Plugin_Name, HTML_Output_File_Data, Title, self.The_File_Extensions["Main_Converted"]) if Output_file and HTML_Output_File: Output_Connections.Output([Output_file, HTML_Output_File], Link, Title, self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def General_Pull(self, Handle, Directory, API): try: Data_to_Cache = [] JSON_Response = [] Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Latest_Tweets = API.user_timeline(screen_name=Handle, count=self.Limit) for Tweet in Latest_Tweets: try: JSON_Response.append({ 'id': Tweet.id, 'text': Tweet.text, 'author_name': Tweet.user.screen_name, 'url': Tweet.entities['urls'][0]["expanded_url"] }) except: JSON_Response.append({ 'id': Tweet.id, 'text': Tweet.text, 'author_name': Tweet.user.screen_name }) JSON_Output = Common.JSON_Handler(JSON_Response).Dump_JSON() Output_Connections = General.Connections(Handle, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) Main_File = General.Main_File_Create(Directory, self.Plugin_Name, JSON_Output, Handle, self.The_File_Extensions["Main"]) for JSON_Item in JSON_Response: if all(Item in JSON_Item for Item in ['id', 'url', 'text']): Link = JSON_Item['url'] if Link not in Cached_Data and Link not in Data_to_Cache: Title = "Twitter | " + JSON_Item['text'] Item_Responses = Common.Request_Handler(Link, Filter=True, Host=f"https://{self.Domain}") Item_Response = Item_Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File(Directory, Handle, self.Plugin_Name, Item_Response, str(JSON_Item['id']), self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], Link, Title, self.Plugin_Name.lower()) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Output file not returned.") else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Insufficient parameters provided.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Convert_to_JSON(Data): Data = str(Data) Flickr_Regex = Common.Regex_Handler(Data, Custom_Regex=r"\[(.+)\]") if Flickr_Regex: New_Data = Flickr_Regex.group(1).replace("...", "").replace("id=b", "'id': ").replace("title=b", "'title': ").replace("(", "{").replace(")", "}").replace("\'}", "}").replace("}", "\'}") New_Data = New_Data.replace("Photo", "") New_Data = f"[{New_Data}]" New_Data = eval(New_Data) JSON_Object = Common.JSON_Handler(New_Data) New_Data = JSON_Object.Dump_JSON() return New_Data
def Search(Query_List, Task_ID): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) for Query in Query_List: Headers_Custom = {"Referer": f"https://www.doingbusiness.org/en/data/exploreeconomies/{Query}"} Main_URL = f"https://wbgindicatorsqa.azure-api.net/DoingBusiness/api/GetEconomyByURL/{Query}" Doing_Business_Response = Common.Request_Handler(Main_URL, Optional_Headers=Headers_Custom) JSON_Object = Common.JSON_Handler(Doing_Business_Response) JSON_Response = JSON_Object.To_JSON_Loads() JSON_Output_Response = JSON_Object.Dump_JSON() if 'message' not in JSON_Response: Main_File = General.Main_File_Create(Directory, Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"]) Item_URL = f"https://www.{Domain}/en/data/exploreeconomies/{Query}" Title = f"Doing Business | {Query}" Current_Doing_Business_Responses = Common.Request_Handler(Item_URL, Filter=True, Host=f"https://www.{Domain}") Current_Doing_Business_Response = Current_Doing_Business_Responses["Filtered"] if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Current_Doing_Business_Response, Query, The_File_Extensions["Query"]) if Output_file: Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Economic Details", Task_ID, Concat_Plugin_Name) Output_Connections.Output([Main_File, Output_file], Item_URL, Title, Concat_Plugin_Name) Data_to_Cache.append(Item_URL) else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: Response = Common.Request_Handler('https://tpbc.herokuapp.com/search/' + Query.replace(" ", "+") + '/?sort=seeds_desc') JSON_Object = Common.JSON_Handler(Response) Response = JSON_Object.To_JSON_Loads() JSON_Response = JSON_Object.Dump_JSON() Output_file = General.Main_File_Create(Directory, Plugin_Name, JSON_Response, Query, The_File_Extension) if Output_file: Current_Step = 0 Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Torrent", Task_ID, Plugin_Name.lower()) for Search_Result in Response: Result_Title = Search_Result["title"] Result_URL = Search_Result["magnet"] if Result_URL not in Cached_Data and Result_URL not in Data_to_Cache and Current_Step < int(Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, JSON_Response, Result_Title, The_File_Extension) if Output_file: Output_Connections.Output([Output_file], Result_URL, General.Get_Title(Result_URL), Plugin_Name.lower()) Data_to_Cache.append(Result_URL) else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") Current_Step += 1 Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Load_Configuration(): logging.info( f"{Common.Date()} - {__name__.strip('plugins.')} - Loading configuration data." ) try: with open(Common.Set_Configuration_File()) as JSON_File: JSON_Object = Common.JSON_Handler(JSON_File) Configuration_Data = JSON_Object.To_JSON_Load() Pinterest_Details = Configuration_Data["inputs"][ Plugin_Name.lower()] if Pinterest_Details['oauth_token']: return Pinterest_Details['oauth_token'] else: return None except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to load location details." )
def Search(Query_List, Task_ID): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Email_Rep_API_Key = Load_Configuration() for Query in Query_List: if Common.Regex_Handler(Query, Type="Email"): API = EmailRep(Email_Rep_API_Key) JSON_Output_Response = API.query(Query) Link = f"https://{Domain}/{Query}" JSON_Object = Common.JSON_Handler(JSON_Output_Response) JSON_Output_Response = JSON_Object.Dump_JSON() JSON_Response = JSON_Object.To_JSON_Loads() if JSON_Response["reputation"] != "none": Table_JSON = {} for Key, Value in JSON_Response.items(): if Key != "details": Table_JSON[Key] = Value else: for Det_Key, Det_Val in JSON_Response["details"].items(): Table_JSON[Det_Key] = Det_Val Filter_JSON = [Table_JSON] Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Email Information", Task_ID, Concat_Plugin_Name) if Query not in Cached_Data and Query not in Data_to_Cache: Responses = Common.Request_Handler(Link, Filter=True, Host=f"https://{Domain}") Filtered_Response = Responses["Filtered"] Title = f"Email Reputation | {Query}" Main_File = General.Main_File_Create(Directory, Concat_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File(Directory, Query, Concat_Plugin_Name, Filtered_Response, Title, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], Link, Title, Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Shodan_API_Key = Load_Configuration() API_Session = PyHunter(Shodan_API_Key) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: try: if Type == "Domain": if Common.Regex_Handler(Query, Type="Domain"): Local_Plugin_Name = Plugin_Name + "-Domain" API_Response = API_Session.domain_search(Query) JSON_Object = Common.JSON_Handler(API_Response) JSON_Output_Response = JSON_Object.Dump_JSON() if API_Response["domain"] and API_Response['emails']: Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Account", Task_ID, Plugin_Name.lower()) Current_Step = 0 for Hunter_Item in API_Response["emails"]: Current_Email_Address = Hunter_Item["value"] Current_Hunter_Item_Host = f"https://{Domain}/verify/{Current_Email_Address}" Current_Hunter_Item_Responses = Common.Request_Handler( Current_Hunter_Item_Host, Filter=True, Host=f"https://{Domain}") Filtered_Response = Current_Hunter_Item_Responses[ "Filtered"] Title = "Hunter | " + Current_Email_Address if Current_Email_Address not in Cached_Data and Current_Email_Address not in Data_to_Cache and Current_Step < int( Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Filtered_Response, Current_Hunter_Item_Host, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Current_Hunter_Item_Host, Title, Plugin_Name.lower()) Data_to_Cache.append( Current_Email_Address) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 elif Type == "Email": if Common.Regex_Handler(Query, Type="Email"): Local_Plugin_Name = Plugin_Name + "-Email" API_Response = API_Session.email_verifier(Query) JSON_Object = Common.JSON_Handler(API_Response) JSON_Output_Response = JSON_Object.Dump_JSON() if API_Response["email"] and API_Response['sources']: Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Account Source", Task_ID, Plugin_Name.lower()) Current_Step = 0 for Hunter_Item in API_Response["sources"]: Current_Hunter_Item_Host = Hunter_Item["uri"] Current_Hunter_Item_Domain = Hunter_Item[ "domain"] if 'http://' in Current_Hunter_Item_Host: Current_Hunter_Item_Responses = Common.Request_Handler( Current_Hunter_Item_Host, Filter=True, Host= f"http://{Current_Hunter_Item_Domain}") Filtered_Response = Current_Hunter_Item_Responses[ "Filtered"] elif 'https://' in Current_Hunter_Item_Host: Current_Hunter_Item_Responses = Common.Request_Handler( Current_Hunter_Item_Host, Filter=True, Host= f"https://{Current_Hunter_Item_Domain}" ) Filtered_Response = Current_Hunter_Item_Responses[ "Filtered"] else: Filtered_Response = Common.Request_Handler( Current_Hunter_Item_Host) Title = "Hunter | " + Current_Hunter_Item_Host if Current_Hunter_Item_Host not in Cached_Data and Current_Hunter_Item_Host not in Data_to_Cache and Current_Step < int( Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Filtered_Response, Current_Hunter_Item_Host, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Current_Hunter_Item_Host, Title, Plugin_Name.lower()) Data_to_Cache.append( Current_Hunter_Item_Host) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to complete task - {str(e)}" ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) print(Directory) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Naver_Details = self.Load_Configuration() Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() if int(self.Limit) > 100: logging.fatal( f"{Common.Date()} - {self.Logging_Plugin_Name} - This plugin does not support limits over 100." ) return None for Query in self.Query_List: URL_Query = urllib.parse.quote(Query) URL = f"https://openapi.{self.Domain}/v1/search/webkr.json?query={URL_Query}&display={str(self.Limit)}&sort=sim" Headers = { "X-Naver-Client-Id": Naver_Details[0], "X-Naver-Client-Secret": Naver_Details[1] } Naver_Response = Common.Request_Handler( URL, Optional_Headers=Headers) JSON_Object = Common.JSON_Handler(Naver_Response) JSON_Response = JSON_Object.To_JSON_Loads() JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, self.Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) if JSON_Response.get('items'): for Naver_Item_Link in JSON_Response['items']: try: if 'title' in Naver_Item_Link and 'link' in Naver_Item_Link: Naver_URL = Naver_Item_Link['link'] Title = Naver_Item_Link['title'] Title = f"Naver | {Title}" if Naver_URL not in Cached_Data and Naver_URL not in Data_to_Cache: Naver_Item_Responses = Common.Request_Handler( Naver_URL, Filter=True, Host=f"https://www.{self.Domain}") Naver_Item_Response = Naver_Item_Responses[ "Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Naver_Item_Response, Naver_URL, self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Naver_URL, Title, self.Plugin_Name.lower()) Data_to_Cache.append(Naver_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}" ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - No results found." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) YouTube_Details = self.Load_Configuration() Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: YouTube_Handler = discovery.build( YouTube_Details[1], YouTube_Details[2], developerKey=YouTube_Details[0], cache_discovery=False) Search_Response = YouTube_Handler.search().list( q=Query, type='video', part='id,snippet', maxResults=self.Limit, ).execute() JSON_Output_Response = Common.JSON_Handler( Search_Response.get('items', [])).Dump_JSON() Main_File = General.Main_File_Create( Directory, self.Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) for Search_Result in Search_Response.get('items', []): Full_Video_URL = f"https://www.{self.Domain}/watch?v=" + Search_Result[ 'id']['videoId'] Search_Video_Responses = Common.Request_Handler( Full_Video_URL, Filter=True, Host=f"https://www.{self.Domain}") Search_Video_Response = Search_Video_Responses["Filtered"] Title = "YouTube | " + Search_Result['snippet']['title'] if Full_Video_URL not in Cached_Data and Full_Video_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Search_Video_Response, Search_Result['id']['videoId'], self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], Full_Video_URL, Title, self.Plugin_Name.lower()) Data_to_Cache.append(Full_Video_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID, Type, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) for Query in Query_List: try: if Type == "CBN": Main_API_URL = f'https://searchapi.mrasservice.ca/Search/api/v1/search?fq=keyword:%7B{Query}%7D+Status_State:Active&lang=en&queryaction=fieldquery&sortfield=Company_Name&sortorder=asc' Response = Common.Request_Handler(Main_API_URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Indented_JSON_Response = JSON_Object.Dump_JSON() Main_Output_File = General.Main_File_Create( Directory, Plugin_Name, Indented_JSON_Response, Query, The_File_Extensions["Main"]) try: if JSON_Response['count'] != 0: Query = str(int(Query)) Main_URL = f'https://{Domain}/search/results?search=%7B{Query}%7D&status=Active' Responses = Common.Request_Handler( Main_URL, Filter=True, Host=f"https://{Domain}") Response = Responses["Filtered"] if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Response, General.Get_Title(Main_URL), The_File_Extensions["Query"]) if Output_file: Output_Connections = General.Connections( Query, Plugin_Name, Domain.strip("beta."), "Company Details", Task_ID, Plugin_Name) Output_Connections.Output( [Main_Output_File, Output_file], Main_URL, f"Canadian Business Number {Query}", Concat_Plugin_Name) Data_to_Cache.append(Main_URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid query provided for CBN Search." ) elif Type == "CCN": Total_Results = 0 Iterator = "page=0" while Limit > Total_Results and Iterator is not None: Main_URL = 'https://searchapi.mrasservice.ca/Search/api/v1/search?fq=keyword:%7B' + urllib.parse.quote( Query ) + f'%7D+Status_State:Active&lang=en&queryaction=fieldquery&sortfield=Company_Name&sortorder=asc&{Iterator}' Response = Common.Request_Handler(Main_URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Total_Results += len(JSON_Response["docs"]) if "paging" in JSON_Response and "next" in JSON_Response.get( "paging"): Iterator = JSON_Response["paging"]["next"] else: Iterator = None Indented_JSON_Response = JSON_Object.Dump_JSON() Limit = General.Get_Limit(Limit) try: Main_File = General.Main_File_Create( Directory, Plugin_Name, Indented_JSON_Response, Query, The_File_Extensions["Main"]) Current_Step = 0 Output_Connections = General.Connections( Query, Plugin_Name, Domain.strip("beta."), "Company Details", Task_ID, Plugin_Name) for JSON_Item in JSON_Response['docs']: if JSON_Item.get('BN'): CCN = JSON_Item['Company_Name'] CBN = str(int(JSON_Item['BN'])) Full_CCN_URL = f'https://{Domain}/search/results?search=%7B{CBN}%7D&status=Active' if Full_CCN_URL not in Cached_Data and Full_CCN_URL not in Data_to_Cache and Current_Step < int( Limit): Current_Responses = Common.Request_Handler( Full_CCN_URL, Filter=True, Host=f"https://{Domain}") Current_Response = Current_Responses[ "Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, str(Current_Response), CCN.replace(' ', '-'), The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Full_CCN_URL, f"Canadian Business Number {CBN} for Query {Query}", Concat_Plugin_Name) Data_to_Cache.append(Full_CCN_URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Unable to retrieve business numbers from the JSON response." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid query provided for CCN Search." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid request type." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to make request." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: if Type == "User": from instagramy import InstagramUser Local_Plugin_Name = Plugin_Name + "-" + Type CSE_Response = InstagramUser(Query) JSON_Object = Common.JSON_Handler(vars(CSE_Response)) CSE_JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, The_File_Extensions["Main"]) if not CSE_Response.is_private: Posts = CSE_Response.posts Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Social Media - Person", Task_ID, Local_Plugin_Name.lower()) Current_Step = 0 for Post in Posts: URL = Post['post_url'] Shortcode = URL.replace(f"https://www.{Domain}/p/", "").replace("/", "") Title = "IG | " + General.Get_Title(URL, Requests=True) if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}") Response = Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Response, Shortcode, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.info( f"{Common.Date()} - {__name__.strip('plugins.')} - The provided user's profile is private and cannot be scraped." ) elif Type == "Tag": from instagramy import InstagramHashTag Local_Plugin_Name = Plugin_Name + "-" + Type CSE_Response = InstagramHashTag(Query) JSON_Object = Common.JSON_Handler(vars(CSE_Response)) CSE_JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, The_File_Extensions["Main"]) Posts = vars( CSE_Response)['tag_data']['edge_hashtag_to_media']['edges'] Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Social Media - Person", Task_ID, Local_Plugin_Name.lower()) Current_Step = 0 for Post in Posts: Shortcode = Post['node']['shortcode'] URL = f"https://www.{Domain}/p/{Shortcode}/" Title = "IG | " + General.Get_Title(URL, Requests=True) if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}") Response = Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Response, Shortcode, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Current_Step += 1 elif Type == "Post": from instagramy import InstagramPost Local_Plugin_Name = Plugin_Name + "-" + Type CSE_Response = InstagramPost(Query) JSON_Object = Common.JSON_Handler(vars(CSE_Response)) CSE_JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Social Media - Place", Task_ID, Local_Plugin_Name.lower()) URL = CSE_Response.url Shortcode = URL.replace(f"https://www.{Domain}/p/", "").replace("/", "") Title = "IG | " + General.Get_Title(URL, Requests=True) if URL not in Cached_Data and URL not in Data_to_Cache: Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}") Response = Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Response, Shortcode, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid type provided." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) for Query in Query_List: Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Vehicle Details", Task_ID, Concat_Plugin_Name) for State in States: Post_URL = f'https://{Domain}/bin/wesfarmers/search/vehiclerego' data = '''{"isRegoSearch":"YES","regoSearchCount":2,"regoMatchCount":1,"regoSearchFailureCount":0,"failPaymentAttempts":0,"pauseStep":"false","campaignBaseURL":"https://secure.colesinsurance.com.au/campaignimages/","sessionState":"OPEN","sessionStep":"0","policyHolders":[],"updateSessionURL":"http://dev.gtw.gp-mdl.auiag.corp:9000/sys/colessessionservice/motor/v1/update-session","insuranceType":"COMP","startDate":"03/07/2019","drivers":[{"driverRef":"MainDriver","yearsLicenced":{"vehRef":"veh1"}}],"priceBeatAttemptsRemaining":"2","currentInsurerOptions":[{"id":"AAMI","value":"AAMI","text":"AAMI"},{"id":"Allianz","value":"Allianz","text":"Allianz"},{"id":"Apia","value":"Apia","text":"Apia"},{"id":"Bingle","value":"Bingle","text":"Bingle"},{"id":"Broker","value":"Broker","text":"Broker"},{"id":"BudgDirect","value":"BudgDirect","text":"Budget Direct"},{"id":"Buzz","value":"Buzz","text":"Buzz"},{"id":"CGU","value":"CGU","text":"CGU"},{"id":"Coles","value":"Coles","text":"Coles"},{"id":"CommInsure","value":"CommInsure","text":"CommInsure"},{"id":"GIO","value":"GIO","text":"GIO"},{"id":"HBF","value":"HBF","text":"HBF"},{"id":"JustCar","value":"JustCar","text":"Just Car"},{"id":"NRMA","value":"NRMA","text":"NRMA"},{"id":"Progress","value":"Progress","text":"Progressive"},{"id":"QBE","value":"QBE","text":"QBE"},{"id":"RAA","value":"RAA","text":"RAA"},{"id":"RAC","value":"RAC","text":"RAC"},{"id":"RACQ","value":"RACQ","text":"RACQ"},{"id":"RACT","value":"RACT","text":"RACT"},{"id":"RACV","value":"RACV","text":"RACV"},{"id":"Real","value":"Real","text":"Real"},{"id":"SGIC","value":"SGIC","text":"SGIC"},{"id":"SGIO","value":"SGIO","text":"SGIO"},{"id":"Shannons","value":"Shannons","text":"Shannons"},{"id":"Suncorp","value":"Suncorp","text":"Suncorp"},{"id":"Youi","value":"Youi","text":"Youi"},{"id":"None","value":"None","text":"Car is not currently insured"},{"id":"Dontknow","value":"Dontknow","text":"Don't Know"},{"id":"Other","value":"Other","text":"Other"}],"coverLevelOptions":[{"id":"Gold","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"NRMA","code":"Gold","order":"1"},{"id":"Gold1","value":"Gold Comprehensive Car Insurance","text":"Gold Comprehensive Car Insurance","flagname":"BudgDirect","code":"Gold","order":"1"},{"id":"Standard2","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"SGIC","code":"Standard","order":"2"},{"id":"Gold6","value":"Comprehensive Advantages Car Insurance","text":"Comprehensive Advantages Car Insurance","flagname":"Suncorp","code":"Gold","order":"1"},{"id":"Standard","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"GIO","code":"Standard","order":"2"},{"id":"Standard0","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"NRMA","code":"Standard","order":"2"},{"id":"Gold4","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"SGIC","code":"Gold","order":"1"},{"id":"Standard5","value":"Full Comprehensive Car Insurance","text":"Full Comprehensive Car Insurance","flagname":"1300 Insurance","code":"Standard","order":"2"},{"id":"Gold5","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"SGIO","code":"Gold","order":"1"},{"id":"Gold2","value":"Platinum Car Insurance","text":"Platinum Car Insurance","flagname":"GIO","code":"Gold","order":"1"},{"id":"Standard3","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"SGIO","code":"Standard","order":"2"},{"id":"Gold3","value":"Complete Care Motor Insurance","text":"Complete Care Motor Insurance","flagname":"RACV","code":"Gold","order":"1"},{"id":"Standard4","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"Suncorp","code":"Standard","order":"2"},{"id":"Gold0","value":"Gold Comprehensive Car Insurance","text":"Gold Comprehensive Car Insurance","flagname":"1300 Insurance","code":"Gold","order":"1"},{"id":"Standard1","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"RACV","code":"Standard","order":"2"}],"riskAddress":{"latitude":"-33.86579240","locality":"PYRMONT","postcode":"2009","extraAddressInfo":{"houseNumber":"1","lotNumber":"1","streetName":"HARRIS","streetSuffix":"STREET","unitNumber":"1"},"state":"''' + State + '''","line3":null,"isVerificationRequired":null,"gnaf":"GANSW709981139","line2":null,"line1":"1 Harris Street","longitude":"151.19109690","displayString":"1 HARRIS STREET, PYRMONT, NSW, 2009"},"postcode":{"latitude":"-33.86579240","locality":"PYRMONT","postcode":"2009","extraAddressInfo":{"houseNumber":"1","lotNumber":"1","streetName":"HARRIS","streetSuffix":"STREET","unitNumber":"1"},"state":"''' + State + '''","line3":null,"isVerificationRequired":null,"gnaf":"GANSW709981139","line2":null,"line1":"1 Harris Street","longitude":"151.19109690","displayString":"1 HARRIS STREET, PYRMONT, NSW, 2009"},"carRegistration":"''' + Query + '''","chooseValue":"","whatValueInsure":"Marketvalue","whatValueInsure_value":{"key":"Marketvalue","value":"Market Value"}}''' headers = { 'Content-Type': 'ext/plain;charset=UTF-8', 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate, br', 'Referer': f'https://{Domain}/motor/get-quote', 'Origin': f'https://{Domain}', 'Host': Domain } Registration_Response = Common.Request_Handler( Post_URL, Method="POST", Data=data, Optional_Headers=headers) JSON_Object = Common.JSON_Handler(Registration_Response) Registration_Response = JSON_Object.To_JSON_Loads() Indented_JSON_Response = JSON_Object.Dump_JSON() try: Title = "Vehicle Registration | " + Registration_Response[ 'vehicles'][0]['make'] + " " + Registration_Response[ 'vehicles'][0]['model'] Item_URL = Post_URL + "?" + Query if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, Indented_JSON_Response, Title.replace(" ", "-"), The_File_Extensions["Main"]) HTML_Output_File_Data = General.JSONDict_to_HTML( Registration_Response["vehicles"], Indented_JSON_Response, f"Vehicle Registration Query {Query}") HTML_Output_File = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, HTML_Output_File_Data, Title.replace(" ", "-"), The_File_Extensions["Main_Converted"]) if Output_file and HTML_Output_File: Output_Connections.Output( [Output_file, HTML_Output_File], Item_URL, Title, Concat_Plugin_Name) Data_to_Cache.append(Item_URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) except: logging.info( f"{Common.Date()} - {__name__.strip('plugins.')} - No result found for given query {Query} for state {State}." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Yandex_Details = Load_Configuration() Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: Yandex_Response = Common.Request_Handler(f"https://{Domain}/search/xml?user={Yandex_Details[0]}&key={Yandex_Details[1]}&query={Query}&l10n=en&sortby=rlv&filter=none&maxpassages=five&groupby=attr% 3D% 22% 22.mode% 3Dflat.groups-on-page% 3D{str(Limit)}.docs-in-group% 3D1") JSON_Response = xmltodict.parse(Yandex_Response) JSON_Object = Common.JSON_Handler(JSON_Response) JSON_Output_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create(Directory, Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"]) Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Search Result", Task_ID, Plugin_Name.lower()) New_JSON_Response = Recursive_Dict_Check(["yandexsearch", "response", "results", "grouping", "group"], JSON_Response) if New_JSON_Response: for Yandex_Item_Line in New_JSON_Response: try: if Recursive_Dict_Check(["doc", "url"], Yandex_Item_Line): Yandex_Item_Line = Yandex_Item_Line['doc'] Yandex_URL = Yandex_Item_Line['url'] Title = Recursive_Dict_Check(["title", "#text"], JSON_Response) if Title: Title = f"Yandex | {Title}" else: Title = General.Get_Title(Yandex_URL) Title = f"Yandex | {Title}" if Yandex_URL not in Cached_Data and Yandex_URL not in Data_to_Cache: Yandex_Item_Responses = Common.Request_Handler(Yandex_URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://{Domain}") Yandex_Item_Response = Yandex_Item_Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Yandex_Item_Response, Yandex_URL, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File, Output_file], Yandex_URL, Title, Plugin_Name.lower()) Data_to_Cache.append(Yandex_URL) else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") except Exception as e: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}") else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - No results found.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: if self.Type == "Email": if Common.Regex_Handler(Query, Type=self.Type): Local_Plugin_Name = self.Plugin_Name + " " + self.Type URL = f"https://www.threatcrowd.org/searchApi/v2/email/report/?email={Query}" Response = Common.Request_Handler(URL) Search_Response = Common.Request_Handler(Request_URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() if int(JSON_Response.get("response_code")) != 0: JSON_Output_Response = JSON_Object.Dump_JSON() Permalink = JSON_Response.get("permalink") Permalink_Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Permalink_Response = Permalink_Responses[ "Filtered"] Title = "Threat Crowd | " + General.Get_Title( Permalink, Requests=True ).replace( " | Threatcrowd.org Open Source Threat Intelligence", "") Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Permalink_Response, Query, self.The_File_Extensions["Query"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, self.Domain, "Account", self.Task_ID, Local_Plugin_Name.lower()) if Output_file: Output_Connections.Output( [Main_File, Output_file], Permalink, Title, self.Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Provided query returned no results." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match query to email regular expression." ) elif self.Type == "Domain": if Common.Regex_Handler(Query, Type=self.Type): Local_Plugin_Name = self.Plugin_Name + " " + self.Type URL = f"https://www.threatcrowd.org/searchApi/v2/self.Domain/report/?self.Domain={Query}" Response = Common.Request_Handler(URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() if int(JSON_Response.get("response_code")) != 0: JSON_Output_Response = JSON_Object.Dump_JSON() Permalink = JSON_Response.get("permalink") Permalink_Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Permalink_Response = Permalink_Responses[ "Filtered"] Title = "Threat Crowd | " + General.Get_Title( Permalink, Requests=True ).replace( " | Threatcrowd.org Open Source Threat Intelligence", "") Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Permalink_Response, Query, self.The_File_Extensions["Query"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, self.Domain, "Domain Information", self.Task_ID, Local_Plugin_Name.lower()) if Output_file: Output_Connections.Output( [Main_File, Output_file], Permalink, Title, self.Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Provided query returned no results." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match query to self.Domain regular expression." ) elif self.Type == "IP Address": if Common.Regex_Handler(Query, Type="IP"): Local_Plugin_Name = self.Plugin_Name + " " + self.Type URL = f"https://www.threatcrowd.org/searchApi/v2/ip/report/?ip={Query}" Response = Common.Request_Handler(URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() if int(JSON_Response.get("response_code")) != 0: JSON_Output_Response = JSON_Object.Dump_JSON() Permalink = JSON_Response.get("permalink") Permalink_Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Permalink_Response = Permalink_Responses[ "Filtered"] Title = "Threat Crowd | " + General.Get_Title( Permalink, Requests=True ).replace( " | Threatcrowd.org Open Source Threat Intelligence", "") Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Permalink_Response, Query, self.The_File_Extensions["Query"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, self.Domain, "Domain Information", self.Task_ID, Local_Plugin_Name.lower()) if Output_file: Output_Connections.Output( [Main_File, Output_file], Permalink, Title, self.Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Provided query returned no results." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match query to IP address regular expression." ) elif self.Type == "AV": Local_Plugin_Name = self.Plugin_Name + " " + self.Type URL = f"https://www.threatcrowd.org/searchApi/v2/antivirus/report/?antivirus={Query}" Response = Common.Request_Handler(URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() if int(JSON_Response.get("response_code")) != 0: JSON_Output_Response = JSON_Object.Dump_JSON() Permalink = JSON_Response.get("permalink") Permalink_Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Permalink_Response = Permalink_Responses["Filtered"] Title = "Threat Crowd | " + General.Get_Title( Permalink, Requests=True ).replace( " | Threatcrowd.org Open Source Threat Intelligence", "") Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Permalink_Response, Query, self.The_File_Extensions["Query"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, self.Domain, "Virus", self.Task_ID, Local_Plugin_Name.lower()) if Output_file: Output_Connections.Output([Main_File, Output_file], Permalink, Title, self.Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Provided query returned no results." ) elif self.Type == "Virus Report": Local_Plugin_Name = self.Plugin_Name + " " + self.Type URL = f"https://www.threatcrowd.org/searchApi/v2/file/report/?resource={Query}" Response = Common.Request_Handler(URL) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() if int(JSON_Response.get("response_code")) != 0: JSON_Output_Response = JSON_Object.Dump_JSON() Permalink = JSON_Response.get("permalink") Permalink_Responses = Common.Request_Handler( URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Permalink_Response = Permalink_Responses["Filtered"] Title = "Threat Crowd | " + General.Get_Title( Permalink, Requests=True ).replace( " | Threatcrowd.org Open Source Threat Intelligence", "") Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Permalink_Response, Query, self.The_File_Extensions["Query"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, self.Domain, "Virus Report", self.Task_ID, Local_Plugin_Name.lower()) if Output_file: Output_Connections.Output([Main_File, Output_file], Permalink, Title, self.Plugin_Name.lower()) Data_to_Cache.append(URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Provided query returned no results." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid Type provided." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID, Type, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: if Type == "pin": Local_Plugin_Name = Plugin_Name + "-" + Type Request_URL = f"https://api.{Domain}/v1/pins/{Query}/?access_token=" + Load_Configuration( ) + "&fields=id%2Clink%2Cnote%2Curl%2Ccreated_at%2Cmedia%2Coriginal_link%2Cmetadata%2Ccounts%2Ccolor%2Cboard%2Cattribution" Search_Response = Common.Request_Handler(Request_URL) JSON_Object = Common.JSON_Handler(Search_Response) Search_Response = JSON_Object.To_JSON_Loads() if Search_Response.get( 'message' ) != "You have exceeded your rate limit. Try again later.": JSON_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Plugin_Name, JSON_Response, Query, The_File_Extensions["Main"]) Result_Title = "Pinterest | " + Search_Response["data"][ "metadata"]["link"]["title"] Result_URL = Search_Response["data"]["url"] Search_Result_Response = Common.Request_Handler(Result_URL) if Result_URL not in Cached_Data and Result_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Search_Result_Response, Result_Title, The_File_Extensions["Query"]) if Output_file: Output_Connections = General.Connections( Query, Local_Plugin_Name, Domain, "Social Media - Media", Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Main_File, Output_file], Result_URL, Result_Title, Plugin_Name.lower()) Data_to_Cache.append(Result_URL) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) elif Type == "board": Local_Plugin_Name = Plugin_Name + "-" + Type Request_URL = "https://api.pinterest.com/v1/boards/" + Query + "/pins/?access_token=" + Load_Configuration( ) + "&fields=id%2Clink%2Cnote%2Curl%2Coriginal_link%2Cmetadata%2Cmedia%2Cimage%2Ccreator%2Ccreated_at%2Ccounts%2Ccolor%2Cboard%2Cattribution&limit=" + str( Limit) + "" Search_Response = Common.Request_Handler(Request_URL) JSON_Object = Common.JSON_Handler(Search_Response) Search_Response = JSON_Object.To_JSON_Loads() if Search_Response.get( 'message' ) != "You have exceeded your rate limit. Try again later.": JSON_Response = JSON_Object.Dump_JSON() Main_File = General.Main_File_Create( Directory, Plugin_Name, JSON_Response, Query, The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, "pinterest.com", "Social Media - Page", Task_ID, Local_Plugin_Name.lower()) Current_Step = 0 for Response in Search_Response["data"]: Result_Title = "Pinterest | " + Response["note"] Result_URL = Response["url"] Search_Result_Response = Common.Request_Handler( Result_URL) if Result_URL not in Cached_Data and Result_URL not in Data_to_Cache and Current_Step < int( Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Search_Result_Response, Result_Title, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Result_URL, Result_Title, Plugin_Name.lower()) Data_to_Cache.append(Result_URL) Current_Step += 1 else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: try: Pull_URL = f"https://{Domain}:2096/api/phishing?_where=(url,like,~{Query}~)&_sort=-id&_size={Limit}" JSON_Object = Common.JSON_Handler( Common.Request_Handler(Pull_URL)) Results = JSON_Object.To_JSON_Loads() Indented_Results = JSON_Object.Dump_JSON() Output_Connections = General.Connections( Query, Plugin_Name, Domain, "Phishing", Task_ID, Plugin_Name.lower()) Main_File = General.Main_File_Create( Directory, Plugin_Name, Indented_Results, Query, The_File_Extensions["Main"]) for Result in Results: Current_Link = Result["url"] Current_Domain = Current_Link.strip("https://") Current_Domain = Current_Domain.strip("http://") Current_Domain = Current_Domain.strip("www.") Current_Title = Result["title"] try: Current_Result = Common.Request_Handler( Current_Link, Filter=True, Risky_Plugin=True, Host=Current_Link) Current_Result_Filtered = Current_Result["Filtered"] Response_Regex = Common.Regex_Handler( Current_Result, Custom_Regex=r"\<title\>([^\<\>]+)\<\/title\>") Output_file_Query = Query.replace(" ", "-") if Current_Link not in Cached_Data and Current_Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Output_file_Query, Plugin_Name, Current_Result_Filtered, Current_Domain, The_File_Extensions["Query"]) if Output_file: if Response_Regex: Current_Title = Response_Regex.group(1) Current_Title = Current_Title.strip() Output_Connections.Output( [Main_File, Output_file], Current_Link, Current_Title, Plugin_Name.lower()) else: if not "Phishstats" in Current_Title: Output_Connections.Output( [Main_File, Output_file], Current_Link, Current_Title, Plugin_Name.lower()) else: Output_Connections.Output( [Main_File, Output_file], Current_Link, General.Get_Title(Current_Link), Plugin_Name.lower()) Data_to_Cache.append(Current_Link) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to make request for result, link may no longer be available." ) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to make request." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: try: Pull_URL = f"https://{self.Domain}:2096/api/phishing?_where=(url,like,~{Query}~)&_sort=-id&_size={self.Limit}" JSON_Object = Common.JSON_Handler( Common.Request_Handler(Pull_URL)) Results = JSON_Object.To_JSON_Loads() Indented_Results = JSON_Object.Dump_JSON() Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) Main_File = General.Main_File_Create( Directory, self.Plugin_Name, Indented_Results, Query, self.The_File_Extensions["Main"]) for Result in Results: Current_Link = Result["url"] Current_Domain = urlparse(Current_Link).netloc Current_Title = Result["title"] try: Response = socket.gethostbyname(Current_Domain) except: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to resolve hostname {Current_Domain} to an IP address. Skipping." ) Response = None if Response: Current_Result = Common.Request_Handler( Current_Link, Filter=True, Risky_Plugin=True, Host=Current_Link) Current_Result_Filtered = Current_Result[ "Filtered"] Response_Regex = Common.Regex_Handler( Current_Result, Custom_Regex=r"\<title\>([^\<\>]+)\<\/title\>") Output_file_Query = Query.replace(" ", "-") if Current_Link not in Cached_Data and Current_Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Output_file_Query, self.Plugin_Name, Current_Result_Filtered, Current_Domain, self.The_File_Extensions["Query"]) if Output_file: if Response_Regex: Current_Title = Response_Regex.group(1) Current_Title = Current_Title.strip() Output_Connections.Output( [Main_File, Output_file], Current_Link, Current_Title, self.Plugin_Name.lower()) else: if not "Phishstats" in Current_Title: Output_Connections.Output( [Main_File, Output_file], Current_Link, Current_Title, self.Plugin_Name.lower()) else: Output_Connections.Output( [Main_File, Output_file], Current_Link, General.Get_Title( Current_Link), self.Plugin_Name.lower()) Data_to_Cache.append(Current_Link) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to resolve DNS, this link probably isn't live." ) except: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to make request." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: try: if self.Type == "UKBN": Authorization_Key = self.Load_Configuration() if Authorization_Key: Authorization_Key = "Basic " + Authorization_Key.decode('ascii') headers_auth = {"Authorization": Authorization_Key} Main_URL = f'https://api.{self.Domain}/company/{Query}' Response = Common.Request_Handler(Main_URL, Optional_Headers=headers_auth) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Indented_JSON_Response = JSON_Object.Dump_JSON() try: Query = str(int(Query)) if Response and '{"errors":[{"error":"company-profile-not-found","self.Type":"ch:service"}]}' not in Response: if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache: Current_Company_Number = str(JSON_Response["company_number"]) Result_URL = f'https://beta.{self.Domain}/company/{Current_Company_Number}' Result_Responses = Common.Request_Handler(Result_URL, Filter=True, Host=f"https://beta.{self.Domain}") Result_Response = Result_Responses["Filtered"] UKCN = str(JSON_Response["company_name"]) Main_Output_File = General.Main_File_Create(Directory, self.Plugin_Name, Indented_JSON_Response, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, Result_Response, UKCN, self.The_File_Extensions["Query"]) if Output_file: Output_Connections = General.Connections(Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name) Output_Connections.Output([Main_Output_File, Output_file], Result_URL, f"UK Business Number {Query}", self.Concat_Plugin_Name) Data_to_Cache.append(Main_URL) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") except: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid query provided for UKBN Search.") else: logging.info(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to retrieve API key.") elif self.Type == "UKCN": Authorization_Key = self.Load_Configuration() if Authorization_Key: Authorization_Key = "Basic " + Authorization_Key.decode('ascii') try: Main_URL = f'https://api.{self.Domain}/search/companies?q={Query}&items_per_page={self.Limit}' headers_auth = {"Authorization": Authorization_Key} Response = Common.Request_Handler(Main_URL, Optional_Headers=headers_auth) JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Indented_JSON_Response = JSON_Object.Dump_JSON() try: if JSON_Response['total_results'] > 0: Main_Output_File = General.Main_File_Create(Directory, self.Plugin_Name, Indented_JSON_Response, Query, self.The_File_Extensions["Main"]) Output_Connections = General.Connections(Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name) for Item in JSON_Response['items']: UKBN_URL = Item['links']['self'] Full_UKBN_URL = f'https://beta.{self.Domain}{str(UKBN_URL)}' UKBN = UKBN_URL.strip("/company/") if Full_UKBN_URL not in Cached_Data and Full_UKBN_URL not in Data_to_Cache: UKCN = Item['title'] Current_Responses = Common.Request_Handler(Full_UKBN_URL, Filter=True, Host=f"https://beta.{self.Domain}") Current_Response = Current_Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, str(Current_Response), UKCN, self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_Output_File, Output_file], Full_UKBN_URL, f"UK Business Number {UKBN} for Query {Query}", self.Concat_Plugin_Name) Data_to_Cache.append(Full_UKBN_URL) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") except: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Error during UKCN Search, perhaps the rate limit has been exceeded.") except: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid query provided for UKCN Search.") else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to retrieve API key.") else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid request self.Type.") except: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to make request.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() try: DNS_Info = checkdmarc.check_domains(self.Query_List) if len(self.Query_List) > 1: for DNS_Item in DNS_Info: Query = DNS_Item['base_domain'] Output_Dict = Common.JSON_Handler(DNS_Item).Dump_JSON() Link = "https://www." + Query Title = "DNS Information for " + DNS_Item['base_domain'] if Link not in Data_to_Cache and Link not in Cached_Data: Responses = Common.Request_Handler( Link, Filter=True, Host=f"https://www.{Query}") Response = Responses["Filtered"] Main_File = General.Main_File_Create( Directory, self.Plugin_Name, Output_Dict, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Response, Title, self.The_File_Extensions["Query"]) if Output_file: Output_Connections = General.Connections( Query, self.Plugin_Name, Query, self.Result_Type, self.Task_ID, self.Concat_Plugin_Name) Output_Connections.Output( [Main_File, Output_file], Link, Title, self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: Query = DNS_Info['base_domain'] Output_Dict = Common.JSON_Handler(DNS_Item).Dump_JSON() Link = "https://www." + Query Title = "DNS Information for " + Query if Link not in Data_to_Cache and Link not in Cached_Data: Responses = Common.Request_Handler( Link, Filter=True, Host=f"https://www.{Query}") Response = Responses["Filtered"] Main_File = General.Main_File_Create( Directory, self.Plugin_Name, Output_Dict, Query, self.The_File_Extensions["Main"]) Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Response, Title, self.The_File_Extensions["Query"]) if Output_file: Output_Connections = General.Connections( Query, self.Plugin_Name, Query, self.Result_Type, self.Task_ID, self.Concat_Plugin_Name) Output_Connections.Output([Main_File, Output_file], Link, Title, self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) except: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Error retrieving DNS details." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: vulners_api = vulners.Vulners(api_key=Load_Configuration()) Search_Response = vulners_api.search(Query, limit=int(self.Limit)) JSON_Response = Common.JSON_Handler( Search_Response).Dump_JSON() Main_File = General.Main_File_Create( Directory, self.Plugin_Name, JSON_Response, Query, self.The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) for Search_Result in Search_Response: if Search_Result[ "bulletinFamily"] not in self.Unacceptable_Bulletins: Result_Title = Search_Result["title"] Result_URL = Search_Result["vhref"] Search_Result_Responses = Common.Request_Handler( Result_URL, Filter=True, Host=f"https://{self.Domain}") Search_Result_Response = Search_Result_Responses[ "Filtered"] if Result_URL not in Cached_Data and Result_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Search_Result_Response, Result_Title, self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Result_URL, Result_Title, self.Plugin_Name.lower()) Data_to_Cache.append(Result_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Skipping as bulletin type is not supported." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: URL_Components = Common.Regex_Handler(Query, Type="URL", Get_URL_Components=True) if URL_Components: BW_Info = builtwith(Query) if BW_Info: BW_JSON_Output = Common.JSON_Handler( BW_Info).Dump_JSON() Query_Domain = URL_Components["Body"] + URL_Components[ "Extension"] Title = f"Built With | {Query_Domain}" Main_File = General.Main_File_Create( Directory, self.Plugin_Name, BW_JSON_Output, Query_Domain, self.The_File_Extensions["Main"]) BW_Search_URL = f"https://{self.Domain}/{Query_Domain}" Responses = Common.Request_Handler( BW_Search_URL, Filter=True, Host=f"https://{self.Domain}") Response = Responses["Filtered"] Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) if BW_Search_URL not in Cached_Data and BW_Search_URL not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Response, Query, self.The_File_Extensions['Query']) if Output_file: Output_Connections.Output( [Main_File, Output_file], BW_Search_URL, Title, self.Plugin_Name.lower()) Data_to_Cache.append(BW_Search_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to get result for provided query." ) else: logging.info( f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid query provided." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Shodan_API_Key = self.Load_Configuration() API_Session = Shodan(Shodan_API_Key) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: try: if self.Type == "Search": Local_Plugin_Name = self.Plugin_Name + "-Search" try: API_Response = API_Session.search(Query) except Exception as e: logging.error( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}." ) break JSON_Output_Response = Common.JSON_Handler( API_Response).Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) Current_Step = 0 for Shodan_Item in API_Response["matches"]: Shodan_Item_Module = Shodan_Item['_shodan'][ 'module'] Shodan_Item_Module = Shodan_Item_Module.replace( '-simple-new', '') if Shodan_Item_Module.startswith("http"): Shodan_Item_Host = "" Shodan_Item_Port = 0 if 'http' in Shodan_Item: Shodan_Item_Host = Shodan_Item['http'][ 'host'] Shodan_Item_Response = Shodan_Item['http'][ 'html'] elif 'ip_str' in Shodan_Item and 'domains' in Shodan_Item and len( Shodan_Item['domains']) > 0: Shodan_Item_Host = Shodan_Item['domains'][ 0] Shodan_Item_Response = Shodan_Item['data'] elif 'ip_str' in Shodan_Item and 'domains' not in Shodan_Item: Shodan_Item_Host = Shodan_Item['ip_str'] Shodan_Item_Response = Shodan_Item['data'] if Shodan_Item_Host: if 'port' in Shodan_Item_Host: if int(Shodan_Item['port']) not in [ 80, 443 ]: Shodan_Item_Port = Shodan_Item[ 'port'] if Shodan_Item_Port != 0: Shodan_Item_URL = f"{Shodan_Item_Module}://{Shodan_Item_Host}:{str(Shodan_Item_Port)}" else: Shodan_Item_URL = f"{Shodan_Item_Module}://{Shodan_Item_Host}" Title = f"Shodan | {str(Shodan_Item_Host)}" if Shodan_Item_URL not in Cached_Data and Shodan_Item_URL not in Data_to_Cache and Current_Step < int( self.Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, Local_Plugin_Name, Shodan_Item_Response, Shodan_Item_Host, self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Shodan_Item_URL, Title, self.Plugin_Name.lower()) Data_to_Cache.append( Shodan_Item_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) Current_Step += 1 elif self.Type == "Host": Local_Plugin_Name = self.Plugin_Name + "-Host" try: API_Response = API_Session.host(Query) except Exception as e: logging.error( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}." ) break JSON_Output_Response = Common.JSON_Handler( API_Response).Dump_JSON() Main_File = General.Main_File_Create( Directory, Local_Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) Output_Connections = General.Connections( Query, Local_Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) Shodan_URL = f"https://www.{self.Domain}/host/{Query}" Title = f"Shodan | {Query}" if Shodan_URL not in Cached_Data and Shodan_URL not in Data_to_Cache: Shodan_Responses = Common.Request_Handler( Shodan_URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Shodan_Response = Shodan_Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Shodan_Response, Query, self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Shodan_URL, Title, self.Plugin_Name.lower()) Data_to_Cache.append(Shodan_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - No results found." ) except: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to complete task." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: URL_Body = f'https://{self.Domain}' Main_URL = URL_Body + '/' + Query.lower().replace(' ', '-') Responses = Common.Request_Handler( Main_URL, Filter=True, Host=f"https://www.{self.Domain}") Response = Responses["Regular"] Filtered_Response = Responses["Filtered"] Main_File = General.Main_File_Create(Directory, self.Plugin_Name, Filtered_Response, Query, self.The_File_Extension) Regex = Common.Regex_Handler( Response, Custom_Regex= r"\<tr\>\s+\<td\sclass\=\"name\"\>\s+\<a\shref\=\"([\/\d\w\-\+\?\.]+)\"\>([\/\d\w\-\+\?\.\(\)\s\,\;\:\~\`\!\@\#\$\%\^\&\*\[\]\{\}]+)\<\/a\>\s+\<\/td\>", Findall=True) if Regex: Current_Step = 0 Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Concat_Plugin_Name) for URL, Title in Regex: Item_URL = URL_Body + URL Current_Response = Common.Request_Handler(Item_URL) Current_Item_Regex = Common.Regex_Handler( Current_Response, Custom_Regex= r"\<button\sclass\=\"btn\sbtn\-primary\spassword\"\s+data\-data\=\"([\-\d\w\?\/]+)\"\s+data\-toggle\=\"modal\"\s+data\-target\=\"\#modal\"\s+\>show\sme\!\<\/button\>" ) if Current_Item_Regex: try: Detailed_Item_URL = URL_Body + Current_Item_Regex.group( 1) Detailed_Responses = Common.Request_Handler( Item_URL, Filter=True, Host=f"https://www.{self.Domain}") Detailed_Response = Detailed_Responses[ "Regular"] Output_Dict = Common.JSON_Handler( Detailed_Response).Is_JSON() if JSON_Response: Output_Response = "<head><title>" + JSON_Response[ "title"] + "</title></head>\n" Output_Response = Output_Response + JSON_Response[ "data"] else: Output_Response = Detailed_Responses[ "Filtered"] if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int( self.Limit): Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Output_Response, Title, self.The_File_Extension) if Output_file: Output_Connections.Output( [Main_File, Output_file], Item_URL, General.Get_Title(Item_URL), self.Concat_Plugin_Name) Data_to_Cache.append(Item_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) Current_Step += 1 except: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to generate output, may have a blank detailed response." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match regular expression for current result." ) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to match regular expression for provided query." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) IX_Access_Token = Load_Configuration() Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: Data = {"term": Query, "buckets": [], "lookuplevel": 0, "maxresults": Limit, "timeout": 0, "datefrom": "", "dateto": "", "sort": 2, "media": 0, "terminate": []} IX_Response = Common.Request_Handler(f"https://2.{Domain}/intelligent/search?k={IX_Access_Token}", Method="POST", JSON_Data=Data) JSON_Object = Common.JSON_Handler(IX_Response) JSON_Response = JSON_Object.To_JSON_Loads() JSON_Output_Response = JSON_Object.Dump_JSON() Main_File_1 = General.Main_File_Create(Directory, Plugin_Name, JSON_Output_Response, Query + "-Request-1", The_File_Extensions["Main"]) if "id" in JSON_Response: Search_ID = JSON_Response["id"] IX_Response = Common.Request_Handler(f"https://2.{Domain}/intelligent/search/result?k={IX_Access_Token}&id={Search_ID}") JSON_Object = Common.JSON_Handler(IX_Response) JSON_Response = JSON_Object.To_JSON_Loads() JSON_Output_Response = JSON_Object.Dump_JSON() Main_File_2 = General.Main_File_Create(Directory, Plugin_Name, JSON_Output_Response, Query + "-Request-2", The_File_Extensions["Main"]) Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Data Leakage", Task_ID, Plugin_Name.lower()) if "records" in JSON_Response: for IX_Item in JSON_Response["records"]: if "systemid" in IX_Item and "name" in IX_Item: IX_URL = f"https://{Domain}/?did=" + IX_Item['systemid'] if IX_Item["name"] != "": Title = f"IntelligenceX Data Leak | " + IX_Item["name"] else: TItle = "IntelligenceX Data Leak | Untitled Document" if IX_URL not in Cached_Data and IX_URL not in Data_to_Cache: IX_Item_Responses = Common.Request_Handler(IX_URL, Filter=True, Host=f"https://{Domain}") IX_Item_Response = IX_Item_Responses["Filtered"] Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, IX_Item_Response, IX_URL, The_File_Extensions["Query"]) if Output_file: Output_Connections.Output([Main_File_1, Main_File_2, Output_file], IX_URL, Title, Plugin_Name.lower()) Data_to_Cache.append(IX_URL) else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") else: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - No results found.") Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Concat_Plugin_Name) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) try: pyhibp.set_api_key(key=Load_Configuration()) except: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to set API key, make sure it is set in the configuration file.") if self.Type == "email": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_pastes(email_address=Query) logging.info(Query_Response) if Query_Response: Current_Domain = Query_Response[0]["Source"] ID = Query_Response[0]["Id"] Link = f"https://www.{Current_Domain}.com/{ID}" JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, JSON_Query_Response, "email", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_1, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "breach": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_single_breach(breach_name=Query) if Query_Response: Current_Domain = Query_Response["Domain"] Link = f"https://www.{Current_Domain}.com/" JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "breach", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_2, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "password": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pw.is_password_breached(password=Query) logging.info(Query_Response) if Query_Response: Link = f"https://{self.Domain}/Passwords?{Query}" if Link not in Cached_Data and Link not in Data_to_Cache: Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, str(Query_Response), "password", ".txt") if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, self.Domain, self.Result_Type_2, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Link) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Cached_Data_Object.Write_Cache(Data_to_Cache) elif self.Type == "account": Local_Plugin_Name = self.Plugin_Name + " " + self.Type Cached_Data_Object = General.Cache(Directory, Local_Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: Query_Response = pyhibp.get_account_breaches(account=Query, truncate_response=True) if Query_Response: Current_Step = 0 for Response in Query_Response: Current_Response = pyhibp.get_single_breach(breach_name=Response['Name']) JSON_Query_Response = Common.JSON_Handler(Query_Response).Dump_JSON() Link = "https://" + Current_Response['self.Domain'] if Current_Response['self.Domain'] not in Cached_Data and Current_Response['self.Domain'] not in Data_to_Cache and Current_Step < int(self.Limit): Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "account", self.The_File_Extension) if Output_file: Output_Connections = General.Connections(Query, Local_Plugin_Name, Current_Response['self.Domain'], self.Result_Type_1, self.Task_ID, Local_Plugin_Name.lower()) Output_Connections.Output([Output_file], Link, General.Get_Title(Link), self.Concat_Plugin_Name) Data_to_Cache.append(Current_Response['self.Domain']) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.") Current_Step += 1 Cached_Data_Object.Write_Cache(Data_to_Cache) else: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid Type provided.") except Exception as e: logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID, Limit=10): try: Data_to_Cache = [] Directory = General.Make_Directory(Concat_Plugin_Name) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Location = Connectors.Load_Location_Configuration() Cached_Data_Object = General.Cache(Directory, Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(Limit) for Query in Query_List: try: Request_Query = urllib.parse.quote(Query) Main_URL = f"http://{Domain}/search?term={Request_Query}&country={Location}&entity=software&limit={str(Limit)}" Response = Common.Request_Handler(Main_URL) except: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to make request, are you connected to the internet?" ) break JSON_Object = Common.JSON_Handler(Response) JSON_Response = JSON_Object.To_JSON_Loads() Main_File = General.Main_File_Create(Directory, Plugin_Name, JSON_Object.Dump_JSON(), Query, The_File_Extensions["Main"]) if 'resultCount' in JSON_Response: if JSON_Response['resultCount'] > 0: Output_Connections = General.Connections( Query, Plugin_Name, Domain, "Application", Task_ID, Concat_Plugin_Name) for JSON_Object in JSON_Response['results']: JSON_Object_Responses = Common.Request_Handler( JSON_Object['artistViewUrl'], Filter=True, Host=f"https://{Domain}") JSON_Object_Response = JSON_Object_Responses[ "Filtered"] if JSON_Object[ 'artistViewUrl'] not in Cached_Data and JSON_Object[ 'artistViewUrl'] not in Data_to_Cache: Apple_Store_Regex = Common.Regex_Handler( JSON_Object['artistViewUrl'], Custom_Regex=r"https\:\/\/apps\.apple\.com\/" + rf"{Location}" + r"\/developer\/[\w\d\-]+\/(id[\d]{9,10})\?.+") if Apple_Store_Regex: Output_file = General.Create_Query_Results_Output_File( Directory, Query, Plugin_Name, JSON_Object_Response, Apple_Store_Regex.group(1), The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], JSON_Object['artistViewUrl'], General.Get_Title( JSON_Object['artistViewUrl']), Concat_Plugin_Name) Data_to_Cache.append( JSON_Object['artistViewUrl']) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid value provided, value not greater than 0." ) else: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid value." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self): try: Data_to_Cache = [] Directory = General.Make_Directory(self.Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, self.Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Ebay_API_Key = self.Load_Configuration() Cached_Data_Object = General.Cache(Directory, self.Plugin_Name) Cached_Data = Cached_Data_Object.Get_Cache() for Query in self.Query_List: try: API_Request = Connection(appid=Ebay_API_Key, config_file=None) API_Response = API_Request.execute('findItemsAdvanced', {'keywords': Query}) JSON_Output_Response = Common.JSON_Handler( API_Response.dict()).Dump_JSON() JSON_Object = Common.JSON_Handler(API_Response.dict()) JSON_Response = JSON_Object.Dump_JSON(Indentation=0, Sort=False) JSON_Response = JSON_Object.To_JSON_Loads() Main_File = General.Main_File_Create( Directory, self.Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"]) if JSON_Response["ack"] == "Success": Output_Connections = General.Connections( Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower()) Current_Step = 0 for JSON_Line in JSON_Response['searchResult']['item']: Ebay_Item_URL = JSON_Line['viewItemURL'] Title = "Ebay | " + General.Get_Title( Ebay_Item_URL) if Ebay_Item_URL not in Cached_Data and Ebay_Item_URL not in Data_to_Cache and Current_Step < int( self.Limit): Ebay_Item_Regex = Common.Regex_Handler( Ebay_Item_URL, Custom_Regex= r"https\:\/\/www\.ebay\.com\/itm\/([\w\d\-]+)\-\/\d+" ) Ebay_Item_Responses = Common.Request_Handler( Ebay_Item_URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{self.Domain}") Ebay_Item_Response = Ebay_Item_Responses[ "Filtered"] Output_file = General.Create_Query_Results_Output_File( Directory, Query, self.Plugin_Name, Ebay_Item_Response, Ebay_Item_Regex.group(1).rstrip("-"), self.The_File_Extensions["Query"]) if Output_file: Output_Connections.Output( [Main_File, Output_file], Ebay_Item_URL, Title, self.Plugin_Name.lower()) Data_to_Cache.append(Ebay_Item_URL) else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist." ) Current_Step += 1 else: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - No results found." ) except: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to make API call." ) Cached_Data_Object.Write_Cache(Data_to_Cache) except Exception as e: logging.warning( f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")