def Search(Query_List, Task_ID, **kwargs): Data_to_Cache = [] Cached_Data = [] if kwargs.get('Limit'): if int(kwargs["Limit"]) > 0: Limit = kwargs["Limit"] else: Limit = 10 Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) if not Cached_Data: Cached_Data = [] Query_List = General.Convert_to_List(Query_List) for Query in Query_List: Tor_Pull_URL = Tor_General_URL + Query Tor_Scrape_URLs = General.Get_Latest_URLs(Tor_Pull_URL, Tor_Scrape_Regex_URL) if Tor_Scrape_URLs: Output_file = General.Main_File_Create(Directory, Tor_Plugin_Name.lower(), "\n".join(Tor_Scrape_URLs), Query, The_File_Extension) if Output_file: Current_Step = 0 for URL in Tor_Scrape_URLs: if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): General.Connections(Output_file, Query, Tor_Plugin_Name, URL, "ahmia.fl", "Domain Spoof", Task_ID, General.Get_Title(URL), Plugin_Name.lower()) Data_to_Cache.append(URL) Current_Step += 1 else: logging.info(General.Date() + " No Tor links scraped.") I2P_Pull_URL = I2P_General_URL + Query I2P_Scrape_URLs = General.Get_Latest_URLs(I2P_Pull_URL, I2P_Scrape_Regex_URL) if I2P_Scrape_URLs: Output_file = General.Main_File_Create(Directory, I2P_Plugin_Name.lower(), "\n".join(I2P_Scrape_URLs), Query, The_File_Extension) if Output_file: Current_Step = 0 for URL in I2P_Scrape_URLs: if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int( Limit): General.Connections(Output_file, Query, I2P_Plugin_Name, URL, "ahmia.fl", "Domain Spoof", Task_ID, General.Get_Title(URL), Plugin_Name.lower()) Data_to_Cache.append(URL) Current_Step += 1 else: logging.info(General.Date() + " No I2P links scraped.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs): try: Data_to_Cache = [] Directory = General.Make_Directory(Plugin_Name.lower()) logger = logging.getLogger() logger.setLevel(logging.INFO) Log_File = General.Logging(Directory, Plugin_Name.lower()) handler = logging.FileHandler(os.path.join(Directory, Log_File), "w") handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) Cached_Data = General.Get_Cache(Directory, Plugin_Name) Query_List = General.Convert_to_List(Query_List) Limit = General.Get_Limit(kwargs) for Query in Query_List: Tor_Pull_URL = Tor_General_URL + Query Tor_Scrape_URLs = General.Get_Latest_URLs(Tor_Pull_URL, Tor_Scrape_Regex_URL) if Tor_Scrape_URLs: Output_file = General.Main_File_Create(Directory, Tor_Plugin_Name.lower(), "\n".join(Tor_Scrape_URLs), Query, The_File_Extension) if Output_file: Current_Step = 0 Output_Connections = General.Connections(Query, Tor_Plugin_Name, "ahmia.fl", "Darkweb Link", Task_ID, Plugin_Name.lower()) for URL in Tor_Scrape_URLs: if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(Limit): Title = f"Ahmia Tor | {URL}" Output_Connections.Output([Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) Current_Step += 1 else: logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") else: logging.info(f"{General.Date()} - {__name__.strip('plugins.')} - No Tor links scraped.") I2P_Pull_URL = I2P_General_URL + Query I2P_Scrape_URLs = General.Get_Latest_URLs(I2P_Pull_URL, I2P_Scrape_Regex_URL) if I2P_Scrape_URLs: Output_file = General.Main_File_Create(Directory, I2P_Plugin_Name.lower(), "\n".join(I2P_Scrape_URLs), Query, The_File_Extension) if Output_file: Current_Step = 0 Output_Connections = General.Connections(Query, I2P_Plugin_Name, "ahmia.fl", "Darkweb Link", Task_ID, Plugin_Name.lower()) for URL in I2P_Scrape_URLs: if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(Limit): Title = f"Ahmia I2P | {URL}" Output_Connections.Output([Output_file], URL, Title, Plugin_Name.lower()) Data_to_Cache.append(URL) Current_Step += 1 else: logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.") else: logging.info(f"{General.Date()} - {__name__.strip('plugins.')} - No I2P links scraped.") if Cached_Data: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a") else: General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w") except Exception as e: logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")