def Search(Query_List, Task_ID):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if General.Regex_Checker(Query, "IP"):
API_Key = Load_Configuration()
Search_Response = General.Request_Handler(
f"http://api.{Domain}/{Query}?access_key={API_Key}")
JSON_Response = json.loads(Search_Response)
JSON_Output_Response = json.dumps(JSON_Response,
indent=4,
sort_keys=True)
Output_Connections = General.Connections(
Query, Plugin_Name, Domain, "IP Address Information",
Task_ID, Plugin_Name.lower())
if Query not in Cached_Data and Query not in Data_to_Cache:
Result_URL = f"https://{Domain}/?{Query}"
Title = f"IP Stack | {Query}"
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, JSON_Output_Response,
Title, The_File_Extension)
if Output_file:
Output_Connections.Output([Output_file], Result_URL,
Title, Plugin_Name.lower())
Data_to_Cache.append(Result_URL)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if General.Regex_Checker(Query, "Domain"):
Request = f"https://{Domain}/?q={Query}"
Responses = General.Request_Handler(Request,
Accept_XML=True,
Accept_Language_EN_US=True,
Filter=True,
Host=f"https://{Domain}")
Response = Responses["Regular"]
Filtered_Response = Responses["Filtered"]
if "<TD class=\"outer\"><I>None found</I></TD>" not in Response:
if Request not in Cached_Data and Request not in Data_to_Cache:
try:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query,
Plugin_Name.lower(), Filtered_Response,
SSLMate_Regex.group(1), The_File_Extension)
if Output_file:
Output_Connections = General.Connections(
Query, Plugin_Name, Domain, "Certificate",
Task_ID, Plugin_Name.lower())
Output_Connections.Output(
[Output_file], Request,
f"Subdomain Certificate Search for {Query}",
Plugin_Name.lower())
Data_to_Cache.append(Request)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
except:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create file."
)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Query does not exist."
)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression."
)
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if Type == "Email":
if General.Regex_Checker(Query, Type):
Local_Plugin_Name = Plugin_Name + "-" + Type
URL = f"https://www.threatcrowd.org/searchApi/v2/email/report/?email={Query}"
Response = General.Request_Handler(URL)
JSON_Response = json.loads(Response)
if int(JSON_Response.get("response_code")) != 0:
JSON_Output_Response = json.dumps(JSON_Response, indent=4, sort_keys=True)
Permalink = JSON_Response.get("permalink")
Permalink_Responses = General.Request_Handler(URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}")
Permalink_Response = Permalink_Responses["Filtered"]
Title = "Threat Crowd | " + General.Get_Title_Requests_Module(Permalink).replace(" | Threatcrowd.org Open Source Threat Intelligence", "")
Main_File = General.Main_File_Create(Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"])
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, Permalink_Response, Query, The_File_Extensions["Query"])
Output_Connections = General.Connections(Query, Local_Plugin_Name, Domain, "Account", Task_ID, Local_Plugin_Name.lower())
if Output_file:
Output_Connections.Output([Main_File, Output_file], Permalink, Title, Plugin_Name.lower())
Data_to_Cache.append(URL)
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
else:
logging.info(f"{General.Date()} - {__name__.strip('plugins.')} - Provided query returned no results.")
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match query to email regular expression.")
elif Type == "Domain":
if General.Regex_Checker(Query, Type):
Local_Plugin_Name = Plugin_Name + "-" + Type
URL = f"https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={Query}"
Response = General.Request_Handler(URL)
JSON_Response = json.loads(Response)
if int(JSON_Response.get("response_code")) != 0:
JSON_Output_Response = json.dumps(JSON_Response, indent=4, sort_keys=True)
Permalink = JSON_Response.get("permalink")
Permalink_Responses = General.Request_Handler(URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}")
Permalink_Response = Permalink_Responses["Filtered"]
Title = "Threat Crowd | " + General.Get_Title_Requests_Module(Permalink).replace(" | Threatcrowd.org Open Source Threat Intelligence", "")
Main_File = General.Main_File_Create(Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"])
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, Permalink_Response, Query, The_File_Extensions["Query"])
Output_Connections = General.Connections(Query, Local_Plugin_Name, Domain, "Domain Information", Task_ID, Local_Plugin_Name.lower())
if Output_file:
Output_Connections.Output([Main_File, Output_file], Permalink, Title, Plugin_Name.lower())
Data_to_Cache.append(URL)
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
else:
logging.info(f"{General.Date()} - {__name__.strip('plugins.')} - Provided query returned no results.")
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match query to domain regular expression.")
elif Type == "IP Address":
if General.Regex_Checker(Query, "IP"):
Local_Plugin_Name = Plugin_Name + "-" + Type
URL = f"https://www.threatcrowd.org/searchApi/v2/ip/report/?ip={Query}"
Response = General.Request_Handler(URL)
JSON_Response = json.loads(Response)
if int(JSON_Response.get("response_code")) != 0:
JSON_Output_Response = json.dumps(JSON_Response, indent=4, sort_keys=True)
Permalink = JSON_Response.get("permalink")
Permalink_Responses = General.Request_Handler(URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}")
Permalink_Response = Permalink_Responses["Filtered"]
Title = "Threat Crowd | " + General.Get_Title_Requests_Module(Permalink).replace(" | Threatcrowd.org Open Source Threat Intelligence", "")
Main_File = General.Main_File_Create(Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"])
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, Permalink_Response, Query, The_File_Extensions["Query"])
Output_Connections = General.Connections(Query, Local_Plugin_Name, Domain, "Domain Information", Task_ID, Local_Plugin_Name.lower())
if Output_file:
Output_Connections.Output([Main_File, Output_file], Permalink, Title, Plugin_Name.lower())
Data_to_Cache.append(URL)
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
else:
logging.info(f"{General.Date()} - {__name__.strip('plugins.')} - Provided query returned no results.")
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match query to IP address regular expression.")
elif Type == "AV":
Local_Plugin_Name = Plugin_Name + "-" + Type
URL = f"https://www.threatcrowd.org/searchApi/v2/antivirus/report/?antivirus={Query}"
Response = General.Request_Handler(URL)
JSON_Response = json.loads(Response)
if int(JSON_Response.get("response_code")) != 0:
JSON_Output_Response = json.dumps(JSON_Response, indent=4, sort_keys=True)
Permalink = JSON_Response.get("permalink")
Permalink_Responses = General.Request_Handler(URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}")
Permalink_Response = Permalink_Responses["Filtered"]
Title = "Threat Crowd | " + General.Get_Title_Requests_Module(Permalink).replace(" | Threatcrowd.org Open Source Threat Intelligence", "")
Main_File = General.Main_File_Create(Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"])
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, Permalink_Response, Query, The_File_Extensions["Query"])
Output_Connections = General.Connections(Query, Local_Plugin_Name, Domain, "Virus", Task_ID, Local_Plugin_Name.lower())
if Output_file:
Output_Connections.Output([Main_File, Output_file], Permalink, Title, Plugin_Name.lower())
Data_to_Cache.append(URL)
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
else:
logging.info(f"{General.Date()} - {__name__.strip('plugins.')} - Provided query returned no results.")
elif Type == "Virus Report":
Local_Plugin_Name = Plugin_Name + "-" + Type
URL = f"https://www.threatcrowd.org/searchApi/v2/file/report/?resource={Query}"
Response = General.Request_Handler(URL)
JSON_Response = json.loads(Response)
if int(JSON_Response.get("response_code")) != 0:
JSON_Output_Response = json.dumps(JSON_Response, indent=4, sort_keys=True)
Permalink = JSON_Response.get("permalink")
Permalink_Responses = General.Request_Handler(URL, Application_JSON_CT=True, Accept_XML=True, Accept_Language_EN_US=True, Filter=True, Host=f"https://www.{Domain}")
Permalink_Response = Permalink_Responses["Filtered"]
Title = "Threat Crowd | " + General.Get_Title_Requests_Module(Permalink).replace(" | Threatcrowd.org Open Source Threat Intelligence", "")
Main_File = General.Main_File_Create(Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"])
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, Permalink_Response, Query, The_File_Extensions["Query"])
Output_Connections = General.Connections(Query, Local_Plugin_Name, Domain, "Virus Report", Task_ID, Local_Plugin_Name.lower())
if Output_file:
Output_Connections.Output([Main_File, Output_file], Permalink, Title, Plugin_Name.lower())
Data_to_Cache.append(URL)
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
else:
logging.info(f"{General.Date()} - {__name__.strip('plugins.')} - Provided query returned no results.")
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Invalid type provided.")
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID):
try:
Data_to_Cache = []
Subdomains = Load_Configuration()
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if Subdomains:
Request = f'https://api.certspotter.com/v1/issuances?domain={Query}&include_subdomains=true&expand=dns_names&expand=issuer&expand=cert'
else:
Request = f'https://api.certspotter.com/v1/issuances?domain={Query}&expand=dns_names&expand=issuer&expand=cert'
Response = General.Request_Handler(Request)
JSON_Response = json.loads(Response)
if 'exists' not in JSON_Response:
if JSON_Response:
if Request not in Cached_Data and Request not in Data_to_Cache:
try:
if General.Regex_Checker(Query, "Domain"):
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name.lower(), json.dumps(JSON_Response, indent=4, sort_keys=True), SSLMate_Regex.group(1), The_File_Extension)
if Output_file:
Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Certificate", Task_ID, Plugin_Name.lower())
Data_to_Cache.append(Request)
if Subdomains:
Output_Connections.Output([Output_file], Request, f"Subdomain Certificate Search for {Query}", Plugin_Name.lower())
else:
Output_Connections.Output([Output_file], Request, f"Domain Certificate Search for {Query}", Plugin_Name.lower())
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression.")
except:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create file.")
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - No response.")
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Query does not exist.")
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
URL_Regex = General.Regex_Checker(Query, "URL")
if URL_Regex:
BW_Info = builtwith(Query)
if BW_Info:
BW_JSON_Output = json.dumps(BW_Info,
indent=4,
sort_keys=True)
URL_Body = URL_Regex.group(3)
if URL_Regex.group(5) and URL_Regex.group(6):
URL_Extension = URL_Regex.group(4) + URL_Regex.group(
5) + URL_Regex.group(6)
elif URL_Regex.group(5):
URL_Extension = URL_Regex.group(4) + URL_Regex.group(5)
else:
URL_Extension = URL_Regex.group(4)
Query_Domain = URL_Body + URL_Extension
Title = f"Built With | {Query_Domain}"
Main_File = General.Main_File_Create(
Directory, Plugin_Name, BW_JSON_Output, Query_Domain,
The_File_Extensions["Main"])
BW_Search_URL = f"https://{Domain}/{Query_Domain}"
Responses = General.Request_Handler(
BW_Search_URL, Filter=True, Host=f"https://{Domain}")
Response = Responses["Filtered"]
Output_Connections = General.Connections(
Query, Plugin_Name, Domain,
"Web Application Architecture", Task_ID,
Plugin_Name.lower())
if BW_Search_URL not in Cached_Data and BW_Search_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Response, Query,
The_File_Extensions['Query'])
if Output_file:
Output_Connections.Output([Main_File, Output_file],
BW_Search_URL, Title,
Plugin_Name.lower())
Data_to_Cache.append(BW_Search_URL)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
else:
logging.info(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to get result for provided query."
)
else:
logging.info(
f"{General.Date()} - {__name__.strip('plugins.')} - Invalid query provided."
)
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Character_Switch(self, Alphabets, Comprehensive_Search):
try:
Local_Plugin_Name = self.Plugin_Name + "-Character-Switch"
Directory = General.Make_Directory(self.Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Local_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File),
"w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
self.Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name)
logging.info(
f"{General.Date()} {__name__.strip('plugins.')} - Character Switching Selected."
)
self.Query_List = General.Convert_to_List(self.Query_List)
for Query in self.Query_List:
URL_Regex = General.Regex_Checker(Query, "URL")
if URL_Regex:
self.URL_Prefix = URL_Regex.group(1)
self.URL_Body = URL_Regex.group(3)
if URL_Regex.group(5) and URL_Regex.group(6):
self.URL_Extension = URL_Regex.group(
4) + URL_Regex.group(5) + URL_Regex.group(6)
elif URL_Regex.group(5):
self.URL_Extension = URL_Regex.group(
4) + URL_Regex.group(5)
else:
self.URL_Extension = URL_Regex.group(4)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Please provide valid URLs."
)
logging.info(
f'{General.Date()} - Provided domain body - {self.URL_Body}'
)
URL_List = list(self.URL_Body.lower())
Local_Plugin_Name = f"{Local_Plugin_Name}-{Alphabets}"
if Alphabets == "Latin":
if not Comprehensive_Search:
if len(self.URL_Body) > 15:
logging.error(
f"{General.Date()} - {__name__.strip('plugins.')} - The length of the body of the provided query: {Query} is greater than 15 characters in length. Condensed punycode domain fuzzing only allows a maximum of 15 characters."
)
return None
else:
Altered_URLs = Rotor.Search(
URL_List,
English_Upper=False,
Numbers=False,
Special_Characters=False,
Asian=False,
Latin=True,
Middle_Eastern=False,
Native_American=False,
North_African=False,
Latin_Alternatives=True,
Comprehensive=False)
else:
if len(self.URL_Body) > 10:
logging.error(
f"{General.Date()} - {__name__.strip('plugins.')} - The length of the body of the provided query: {Query} is greater than 10 characters in length. Comprehensive punycode domain fuzzing searching only allows a maximum of 10 characters."
)
return None
else:
Altered_URLs = Rotor.Search(
URL_List,
English_Upper=False,
Numbers=False,
Special_Characters=False,
Asian=False,
Latin=True,
Middle_Eastern=False,
Native_American=False,
North_African=False,
Latin_Alternatives=True,
Comprehensive=True)
elif Alphabets == "Asian":
if len(self.URL_Body) > 10:
logging.error(
f"{General.Date()} - {__name__.strip('plugins.')} - The length of the body of the provided query: {Query} is greater than 15 characters in length. Punycode domain fuzzing for Asian alphabets only allows a maximum of 10 characters."
)
return None
else:
Altered_URLs = Rotor.Search(URL_List,
English_Upper=False,
Numbers=False,
Special_Characters=False,
Asian=True,
Latin=False,
Middle_Eastern=False,
Native_American=False,
North_African=False,
Latin_Alternatives=False,
Comprehensive=False)
elif Alphabets == "Middle Eastern":
if len(self.URL_Body) > 10:
logging.error(
f"{General.Date()} - {__name__.strip('plugins.')} - The length of the body of the provided query: {Query} is greater than 15 characters in length. Punycode domain fuzzing for Middle Eastern alphabets only allows a maximum of 10 characters."
)
return None
else:
Altered_URLs = Rotor.Search(URL_List,
English_Upper=False,
Numbers=False,
Special_Characters=False,
Asian=False,
Latin=False,
Middle_Eastern=True,
Native_American=False,
North_African=False,
Latin_Alternatives=False,
Comprehensive=False)
elif Alphabets == "Native American":
if len(self.URL_Body) > 10:
logging.error(
f"{General.Date()} - {__name__.strip('plugins.')} - The length of the body of the provided query: {Query} is greater than 15 characters in length. Punycode domain fuzzing for Asian alphabets only allows a maximum of 10 characters."
)
return None
else:
Altered_URLs = Rotor.Search(URL_List,
English_Upper=False,
Numbers=False,
Special_Characters=False,
Asian=False,
Latin=False,
Middle_Eastern=False,
Native_American=True,
North_African=False,
Latin_Alternatives=False,
Comprehensive=False)
elif Alphabets == "North African":
if len(self.URL_Body) > 10:
logging.error(
f"{General.Date()} - {__name__.strip('plugins.')} - The length of the body of the provided query: {Query} is greater than 15 characters in length. Punycode domain fuzzing for Middle Eastern alphabets only allows a maximum of 10 characters."
)
return None
else:
Altered_URLs = Rotor.Search(URL_List,
English_Upper=False,
Numbers=False,
Special_Characters=False,
Asian=False,
Latin=False,
Middle_Eastern=False,
Native_American=False,
North_African=True,
Latin_Alternatives=False,
Comprehensive=False)
logging.info(
f'{General.Date()} - Generated domain combinations - {", ".join(Altered_URLs)}'
)
Pool = mpool.ThreadPool(
int(multiprocessing.cpu_count()) *
int(multiprocessing.cpu_count()))
Pool_Threads = []
for Altered_URL in Altered_URLs:
if not Altered_URL == self.URL_Body:
Thread = Pool.apply_async(self.Query_URL,
args=(
Altered_URL,
self.URL_Extension,
))
Pool_Threads.append(Thread)
[Pool_Thread.wait() for Pool_Thread in Pool_Threads]
logging.info(f'{General.Date()} {Directory}')
URL_Domain = self.URL_Body + self.URL_Extension
logging.info(URL_Domain)
Main_File = General.Main_File_Create(
Directory, Local_Plugin_Name,
"\n".join(self.Valid_Results), self.URL_Body,
self.The_File_Extensions["Main"])
logging.info(Main_File)
if Main_File:
for Host in self.Valid_Hosts:
Current_Domain = Host[0].strip('https://').strip(
'http://')
try:
Current_Responses = General.Request_Handler(
Host[0],
Filter=True,
Host=Host[0],
Risky_Plugin=True)
Current_Response = Current_Responses["Filtered"]
Output_File = General.Create_Query_Results_Output_File(
Directory, Query, Local_Plugin_Name,
Current_Response, Current_Domain,
self.The_File_Extensions["Query"])
if Output_File:
Output_File_List = [Main_File, Output_File]
Output_Connections = General.Connections(
Query, Local_Plugin_Name, Current_Domain,
"Domain Spoof", self.Task_ID,
Local_Plugin_Name.lower())
Output_Connections.Output(
Output_File_List,
Host[0],
f"Domain Spoof for {URL_Domain} - {Current_Domain} : {Host[1]}",
Directory_Plugin_Name=self.
Concat_Plugin_Name)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
except requests.exceptions.ConnectionError:
Output_File_List = [Main_File]
Output_Connections = General.Connections(
Query, Local_Plugin_Name, Current_Domain,
"Domain Spoof", self.Task_ID,
Local_Plugin_Name.lower())
Output_Connections.Output(
Output_File_List,
Host[0],
f"Domain Spoof for {URL_Domain} - {Current_Domain} : {Host[1]}",
Directory_Plugin_Name=self.Concat_Plugin_Name)
if self.Cached_Data:
General.Write_Cache(Directory, self.Data_to_Cache,
Local_Plugin_Name, "a")
else:
General.Write_Cache(Directory, self.Data_to_Cache,
Local_Plugin_Name, "w")
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def All_Extensions(self):
try:
Local_Plugin_Name = self.Plugin_Name + "-All-Extensions"
Directory = General.Make_Directory(self.Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Local_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File),
"w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
self.Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name)
logging.info(
f"{General.Date()} {__name__.strip('plugins.')} - All Extensions Selected."
)
self.Query_List = General.Convert_to_List(self.Query_List)
for Query in self.Query_List:
URL_Regex = General.Regex_Checker(Query, "URL")
if URL_Regex:
self.URL_Prefix = URL_Regex.group(1)
self.URL_Body = URL_Regex.group(3)
if URL_Regex.group(5) and URL_Regex.group(6):
self.URL_Extension = URL_Regex.group(
4) + URL_Regex.group(5) + URL_Regex.group(6)
elif URL_Regex.group(5):
self.URL_Extension = URL_Regex.group(
4) + URL_Regex.group(5)
else:
self.URL_Extension = URL_Regex.group(4)
else:
logging.warning(
f"{General.Date()} {__name__.strip('plugins.')} - Please provide valid URLs."
)
Pool = mpool.ThreadPool(
int(multiprocessing.cpu_count()) *
int(multiprocessing.cpu_count()))
Pool_Threads = []
for Extension in self.Generic_Extensions:
for suffix in self.Global_Domain_Suffixes:
suffix = suffix.replace(".com", "")
suffix = suffix.replace(".co", "")
if not self.URL_Extension == suffix:
Thread = Pool.apply_async(self.Query_URL,
args=(
self.URL_Body,
Extension + suffix,
))
Pool_Threads.append(Thread)
[Pool_Thread.wait() for Pool_Thread in Pool_Threads]
URL_Domain = self.URL_Body + self.URL_Extension
Main_File = General.Main_File_Create(
Directory, Local_Plugin_Name,
"\n".join(self.Valid_Results), self.URL_Body,
self.The_File_Extensions["Main"])
if Main_File:
for Host in self.Valid_Hosts:
Current_Domain = Host[0].strip('https://').strip(
'http://')
try:
Current_Responses = General.Request_Handler(
Host[0],
Filter=True,
Host=Host[0],
Risky_Plugin=True)
Current_Response = Current_Responses["Filtered"]
Output_File = General.Create_Query_Results_Output_File(
Directory, Query, Local_Plugin_Name,
Current_Response, Current_Domain,
self.The_File_Extensions["Query"])
if Output_File:
Output_File_List = [Main_File, Output_File]
Output_Connections = General.Connections(
Query, Local_Plugin_Name, Current_Domain,
"Domain Spoof", self.Task_ID,
Local_Plugin_Name.lower())
Output_Connections.Output(
Output_File_List,
Host[0],
f"Domain Spoof for {URL_Domain} - {Current_Domain} : {Host[1]}",
Directory_Plugin_Name=self.
Concat_Plugin_Name)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
except requests.exceptions.ConnectionError:
Output_File_List = [Main_File]
Output_Connections = General.Connections(
Query, Local_Plugin_Name, Current_Domain,
"Domain Spoof", self.Task_ID,
Local_Plugin_Name.lower())
Output_Connections.Output(
Output_File_List,
Host[0],
f"Domain Spoof for {URL_Domain} - {Current_Domain} : {Host[1]}",
Directory_Plugin_Name=self.Concat_Plugin_Name)
if self.Data_to_Cache:
if self.Cached_Data:
General.Write_Cache(Directory, self.Data_to_Cache,
Local_Plugin_Name, "a")
else:
General.Write_Cache(Directory, self.Data_to_Cache,
Local_Plugin_Name, "w")
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type, **kwargs):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Shodan_API_Key = Load_Configuration()
API_Session = PyHunter(Shodan_API_Key)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(kwargs)
for Query in Query_List:
try:
if Type == "Domain":
if General.Regex_Checker(Query, "Domain"):
Local_Plugin_Name = Plugin_Name + "-Domain"
API_Response = API_Session.domain_search(Query)
JSON_Output_Response = json.dumps(API_Response, indent=4, sort_keys=True)
if API_Response["domain"] and API_Response['emails']:
Main_File = General.Main_File_Create(Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"])
Output_Connections = General.Connections(Query, Local_Plugin_Name, Domain, "Account", Task_ID, Plugin_Name.lower())
Current_Step = 0
for Hunter_Item in API_Response["emails"]:
Current_Email_Address = Hunter_Item["value"]
Current_Hunter_Item_Host = f"https://{Domain}/verify/{Current_Email_Address}"
Current_Hunter_Item_Responses = General.Request_Handler(Current_Hunter_Item_Host, Filter=True, Host=f"https://{Domain}")
Filtered_Response = Current_Hunter_Item_Responses["Filtered"]
Title = "Hunter | " + Current_Email_Address
if Current_Email_Address not in Cached_Data and Current_Email_Address not in Data_to_Cache and Current_Step < int(Limit):
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, Filtered_Response, Current_Hunter_Item_Host, The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output([Main_File, Output_file], Current_Hunter_Item_Host, Title, Plugin_Name.lower())
Data_to_Cache.append(Current_Email_Address)
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
Current_Step += 1
elif Type == "Email":
if General.Regex_Checker(Query, "Email"):
Local_Plugin_Name = Plugin_Name + "-Email"
API_Response = API_Session.email_verifier(Query)
JSON_Output_Response = json.dumps(API_Response, indent=4, sort_keys=True)
if API_Response["email"] and API_Response['sources']:
Main_File = General.Main_File_Create(Directory, Local_Plugin_Name, JSON_Output_Response, Query, The_File_Extensions["Main"])
Output_Connections = General.Connections(Query, Local_Plugin_Name, Domain, "Account Source", Task_ID, Plugin_Name.lower())
Current_Step = 0
for Hunter_Item in API_Response["sources"]:
Current_Hunter_Item_Host = Hunter_Item["uri"]
Current_Hunter_Item_Domain = Hunter_Item["domain"]
if 'http://' in Current_Hunter_Item_Host:
Current_Hunter_Item_Responses = General.Request_Handler(Current_Hunter_Item_Host, Filter=True, Host=f"http://{Current_Hunter_Item_Domain}")
Filtered_Response = Current_Hunter_Item_Responses["Filtered"]
elif 'https://' in Current_Hunter_Item_Host:
Current_Hunter_Item_Responses = General.Request_Handler(Current_Hunter_Item_Host, Filter=True, Host=f"https://{Current_Hunter_Item_Domain}")
Filtered_Response = Current_Hunter_Item_Responses["Filtered"]
else:
Filtered_Response = General.Request_Handler(Current_Hunter_Item_Host)
Title = "Hunter | " + Current_Hunter_Item_Host
if Current_Hunter_Item_Host not in Cached_Data and Current_Hunter_Item_Host not in Data_to_Cache and Current_Step < int(Limit):
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, Filtered_Response, Current_Hunter_Item_Host, The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output([Main_File, Output_file], Current_Hunter_Item_Host, Title, Plugin_Name.lower())
Data_to_Cache.append(Current_Hunter_Item_Host)
else:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
Current_Step += 1
except Exception as e:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to complete task - {str(e)}")
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
