def Search(Query_List, Task_ID):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data_Object = General.Cache(Directory, Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if Common.Regex_Handler(Query, Type="Email"):
Link = f"https://{Domain}/home/verify-as-guest/{Query}"
JSON_Response = Common.Request_Handler(Link)
JSON_Object = Common.JSON_Handler(JSON_Response)
if JSON_Object.Is_JSON():
JSON_Response = JSON_Object.To_JSON_Loads()
JSON_Output_Response = JSON_Object.Dump_JSON()
Table_JSON = {}
for Key, Value in JSON_Response.items():
if Key != "response":
Table_JSON[Key] = Value
else:
for Det_Key, Det_Val in JSON_Response[
"response"].items():
Table_JSON[Det_Key] = Det_Val
Filter_JSON = [Table_JSON]
Output_Connections = General.Connections(
Query, Plugin_Name, Domain, "Email Information",
Task_ID, Concat_Plugin_Name)
if Query not in Cached_Data and Query not in Data_to_Cache:
Title = f"Email Verification | {Query}"
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Concat_Plugin_Name,
JSON_Output_Response, Title,
The_File_Extensions["Main"])
HTML_Output_File_Data = General.JSONDict_to_HTML(
Filter_JSON, JSON_Output_Response,
f"Email Verification Query {Query}")
HTML_Output_File = General.Create_Query_Results_Output_File(
Directory, Query, Concat_Plugin_Name,
HTML_Output_File_Data, Title,
The_File_Extensions["Main_Converted"])
if Output_file and HTML_Output_File:
Output_Connections.Output(
[Output_file, HTML_Output_File], Link, Title,
Concat_Plugin_Name)
Data_to_Cache.append(Link)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID):
Data_to_Cache = []
Cached_Data = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
for State in States:
Post_URL = 'https://general-insurance.coles.com.au/bin/wesfarmers/search/vehiclerego'
data = '''{"isRegoSearch":"YES","regoSearchCount":2,"regoMatchCount":1,"regoSearchFailureCount":0,"failPaymentAttempts":0,"pauseStep":"false","campaignBaseURL":"https://secure.colesinsurance.com.au/campaignimages/","sessionState":"OPEN","sessionStep":"0","policyHolders":[],"updateSessionURL":"http://dev.gtw.gp-mdl.auiag.corp:9000/sys/colessessionservice/motor/v1/update-session","insuranceType":"COMP","startDate":"03/07/2019","drivers":[{"driverRef":"MainDriver","yearsLicenced":{"vehRef":"veh1"}}],"priceBeatAttemptsRemaining":"2","currentInsurerOptions":[{"id":"AAMI","value":"AAMI","text":"AAMI"},{"id":"Allianz","value":"Allianz","text":"Allianz"},{"id":"Apia","value":"Apia","text":"Apia"},{"id":"Bingle","value":"Bingle","text":"Bingle"},{"id":"Broker","value":"Broker","text":"Broker"},{"id":"BudgDirect","value":"BudgDirect","text":"Budget Direct"},{"id":"Buzz","value":"Buzz","text":"Buzz"},{"id":"CGU","value":"CGU","text":"CGU"},{"id":"Coles","value":"Coles","text":"Coles"},{"id":"CommInsure","value":"CommInsure","text":"CommInsure"},{"id":"GIO","value":"GIO","text":"GIO"},{"id":"HBF","value":"HBF","text":"HBF"},{"id":"JustCar","value":"JustCar","text":"Just Car"},{"id":"NRMA","value":"NRMA","text":"NRMA"},{"id":"Progress","value":"Progress","text":"Progressive"},{"id":"QBE","value":"QBE","text":"QBE"},{"id":"RAA","value":"RAA","text":"RAA"},{"id":"RAC","value":"RAC","text":"RAC"},{"id":"RACQ","value":"RACQ","text":"RACQ"},{"id":"RACT","value":"RACT","text":"RACT"},{"id":"RACV","value":"RACV","text":"RACV"},{"id":"Real","value":"Real","text":"Real"},{"id":"SGIC","value":"SGIC","text":"SGIC"},{"id":"SGIO","value":"SGIO","text":"SGIO"},{"id":"Shannons","value":"Shannons","text":"Shannons"},{"id":"Suncorp","value":"Suncorp","text":"Suncorp"},{"id":"Youi","value":"Youi","text":"Youi"},{"id":"None","value":"None","text":"Car is not currently insured"},{"id":"Dontknow","value":"Dontknow","text":"Don't Know"},{"id":"Other","value":"Other","text":"Other"}],"coverLevelOptions":[{"id":"Gold","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"NRMA","code":"Gold","order":"1"},{"id":"Gold1","value":"Gold Comprehensive Car Insurance","text":"Gold Comprehensive Car Insurance","flagname":"BudgDirect","code":"Gold","order":"1"},{"id":"Standard2","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"SGIC","code":"Standard","order":"2"},{"id":"Gold6","value":"Comprehensive Advantages Car Insurance","text":"Comprehensive Advantages Car Insurance","flagname":"Suncorp","code":"Gold","order":"1"},{"id":"Standard","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"GIO","code":"Standard","order":"2"},{"id":"Standard0","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"NRMA","code":"Standard","order":"2"},{"id":"Gold4","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"SGIC","code":"Gold","order":"1"},{"id":"Standard5","value":"Full Comprehensive Car Insurance","text":"Full Comprehensive Car Insurance","flagname":"1300 Insurance","code":"Standard","order":"2"},{"id":"Gold5","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"SGIO","code":"Gold","order":"1"},{"id":"Gold2","value":"Platinum Car Insurance","text":"Platinum Car Insurance","flagname":"GIO","code":"Gold","order":"1"},{"id":"Standard3","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"SGIO","code":"Standard","order":"2"},{"id":"Gold3","value":"Complete Care Motor Insurance","text":"Complete Care Motor Insurance","flagname":"RACV","code":"Gold","order":"1"},{"id":"Standard4","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"Suncorp","code":"Standard","order":"2"},{"id":"Gold0","value":"Gold Comprehensive Car Insurance","text":"Gold Comprehensive Car Insurance","flagname":"1300 Insurance","code":"Gold","order":"1"},{"id":"Standard1","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"RACV","code":"Standard","order":"2"}],"riskAddress":{"latitude":"-33.86579240","locality":"PYRMONT","postcode":"2009","extraAddressInfo":{"houseNumber":"1","lotNumber":"1","streetName":"HARRIS","streetSuffix":"STREET","unitNumber":"1"},"state":"''' + State + '''","line3":null,"isVerificationRequired":null,"gnaf":"GANSW709981139","line2":null,"line1":"1 Harris Street","longitude":"151.19109690","displayString":"1 HARRIS STREET, PYRMONT, NSW, 2009"},"postcode":{"latitude":"-33.86579240","locality":"PYRMONT","postcode":"2009","extraAddressInfo":{"houseNumber":"1","lotNumber":"1","streetName":"HARRIS","streetSuffix":"STREET","unitNumber":"1"},"state":"''' + State + '''","line3":null,"isVerificationRequired":null,"gnaf":"GANSW709981139","line2":null,"line1":"1 Harris Street","longitude":"151.19109690","displayString":"1 HARRIS STREET, PYRMONT, NSW, 2009"},"carRegistration":"''' + Query + '''","chooseValue":"","whatValueInsure":"Marketvalue","whatValueInsure_value":{"key":"Marketvalue","value":"Market Value"}}'''
headers = {
'Content-Type': 'ext/plain;charset=UTF-8',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Referer':
'https://general-insurance.coles.com.au/motor/get-quote',
'Origin': 'https://general-insurance.coles.com.au',
'Host': 'general-insurance.coles.com.au'
}
Registration_Response = requests.post(Post_URL,
data=data,
headers=headers).text
Registration_Response = json.loads(Registration_Response)
try:
Title = Registration_Response['vehicles'][0][
'make'] + " " + Registration_Response['vehicles'][0][
'model']
Item_URL = Post_URL + "?" + Query
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name,
json.dumps(Registration_Response,
indent=4,
sort_keys=True), Title, The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name,
Item_URL,
"general-insurance.coles.com.au",
"Data Leakage", Task_ID, Title,
Concat_Plugin_Name)
Data_to_Cache.append(Item_URL)
except:
logging.info(
str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+ " No result found for given query " + Query +
" for state " + State + ".")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
Tor_Pull_URL = Tor_General_URL + Query
Tor_Scrape_URLs = General.Get_Latest_URLs(Tor_Pull_URL,
Tor_Scrape_Regex_URL, Is_Tor)
if Tor_Scrape_URLs:
Output_file = General.Main_File_Create(Directory,
Tor_Plugin_Name.lower(),
"\n".join(Tor_Scrape_URLs),
Query, The_File_Extension)
if Output_file:
Current_Step = 0
for URL in Tor_Scrape_URLs:
if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(
Limit):
General.Connections(Output_file, Query,
Tor_Plugin_Name, URL, "ahmia.fl",
"Domain Spoof", Task_ID,
General.Get_Title(URL),
Plugin_Name.lower())
Data_to_Cache.append(URL)
Current_Step += 1
I2P_Pull_URL = I2P_General_URL + Query
I2P_Scrape_URLs = General.Get_Latest_URLs(I2P_Pull_URL,
I2P_Scrape_Regex_URL, Is_Tor)
if I2P_Scrape_URLs:
Output_file = General.Main_File_Create(Directory,
I2P_Plugin_Name.lower(),
"\n".join(Scrape_URLs),
Query, The_File_Extension)
if Output_file:
Current_Step = 0
for URL in I2P_Scrape_URLs:
if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(
Limit):
General.Connections(Output_file, Query,
I2P_Plugin_Name, URL, "ahmia.fl",
"Domain Spoof", Task_ID,
General.Get_Title(URL),
Plugin_Name.lower())
Data_to_Cache.append(URL)
Current_Step += 1
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Yandex_Details = Load_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(kwargs)
for Query in Query_List:
Yandex_Response = requests.get(
f"https://yandex.com/search/xml?user={Yandex_Details[0]}&key={Yandex_Details[1]}&query={Query}&l10n=en&sortby=rlv&filter=none&maxpassages=five&groupby=attr% 3D% 22% 22.mode% 3Dflat.groups-on-page% 3D{str(Limit)}.docs-in-group% 3D1"
).text
JSON_Response = xmltodict.parse(Yandex_Response)
JSON_Output_Response = json.dumps(JSON_Response,
indent=4,
sort_keys=True)
Main_File = General.Main_File_Create(Directory, Plugin_Name,
JSON_Output_Response, Query,
The_File_Extensions["Main"])
Output_Connections = General.Connections(Query, Plugin_Name,
"yandex.ru",
"Search Result", Task_ID,
Plugin_Name.lower())
New_JSON_Response = Recursive_Dict_Check(
["yandexsearch", "response", "results", "grouping", "group"],
JSON_Response)
if New_JSON_Response:
for Yandex_Item_Line in New_JSON_Response:
try:
if Recursive_Dict_Check(["doc", "url"],
Yandex_Item_Line):
Yandex_Item_Line = Yandex_Item_Line['doc']
Yandex_URL = Yandex_Item_Line['url']
Title = Recursive_Dict_Check(["title", "#text"],
JSON_Response)
if Title:
Title = f"Yandex | {Title}"
else:
Title = General.Get_Title(Yandex_URL)
Title = f"Yandex | {Title}"
if Yandex_URL not in Cached_Data and Yandex_URL not in Data_to_Cache:
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Yandex_Item_Response = requests.get(
Yandex_URL, headers=headers).text
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name,
Yandex_Item_Response, Yandex_URL,
The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output(
[Main_File, Output_file], Yandex_URL,
Title, Plugin_Name.lower())
Data_to_Cache.append(Yandex_URL)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}"
)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - No results found."
)
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
Pull_URL = "https://www.phishtank.com/target_search.php?target_id=" + Query + "&valid=y&active=All&Search=Search"
Content = requests.get(Pull_URL).text
soup = BeautifulSoup(Content, features="lxml")
tds = soup.findAll('td')
Links = []
for td in tds:
link = td.find('a')
if link and 'phish_detail.php?phish_id=' in link.attrs['href']:
Full_Link = "https://www.phishtank.com/" + link.attrs['href']
Links.append(Full_Link)
Current_Step = 0
for Link in Links:
Current_Content = requests.get(Link).text
Current_Soup = BeautifulSoup(Current_Content, features="lxml")
Spans = Current_Soup.find('span',
{"style": "word-wrap:break-word;"})
Current_Link = Spans.string
if Current_Link:
Phish_Site_Response = requests.get(Current_Link).text
Output_file_query = Query.replace(" ", "-")
Output_file = General.Create_Query_Results_Output_File(
Directory, Output_file_query, Plugin_Name,
Phish_Site_Response,
Link.replace(
"https://www.phishtank.com/phish_detail.php?phish_id=",
""), The_File_Extension)
if Output_file:
if Current_Link not in Cached_Data and Current_Link not in Data_to_Cache and Current_Step < int(
Limit):
General.Connections(Output_file, Query, Plugin_Name,
Current_Link, "phishtank.com",
"Phishing", Task_ID,
General.Get_Title(Current_Link),
Plugin_Name.lower())
Data_to_Cache.append(Current_Link)
Current_Step += 1
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, Limit=10):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Shodan_API_Key = Load_Configuration()
API_Session = Shodan(Shodan_API_Key)
Cached_Data_Object = General.Cache(Directory, Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(Limit)
for Query in Query_List:
try:
if Type == "Search":
Local_Plugin_Name = Plugin_Name + "-Search"
try:
API_Response = API_Session.search(Query)
except Exception as e:
logging.error(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}."
)
break
JSON_Object = Common.JSON_Handler(API_Response)
JSON_Output_Response = JSON_Object.Dump_JSON()
Main_File = General.Main_File_Create(
Directory, Local_Plugin_Name, JSON_Output_Response,
Query, The_File_Extensions["Main"])
Output_Connections = General.Connections(
Query, Local_Plugin_Name, Domain, "Domain Information",
Task_ID, Plugin_Name.lower())
Current_Step = 0
for Shodan_Item in API_Response["matches"]:
Shodan_Item_Module = Shodan_Item['_shodan']['module']
Shodan_Item_Module = Shodan_Item_Module.replace(
'-simple-new', '')
if Shodan_Item_Module.startswith("http"):
Shodan_Item_Host = ""
Shodan_Item_Port = 0
if 'http' in Shodan_Item:
Shodan_Item_Host = Shodan_Item['http']['host']
Shodan_Item_Response = Shodan_Item['http'][
'html']
elif 'ip_str' in Shodan_Item and 'domains' in Shodan_Item and len(
Shodan_Item['domains']) > 0:
Shodan_Item_Host = Shodan_Item['domains'][0]
Shodan_Item_Response = Shodan_Item['data']
elif 'ip_str' in Shodan_Item and 'domains' not in Shodan_Item:
Shodan_Item_Host = Shodan_Item['ip_str']
Shodan_Item_Response = Shodan_Item['data']
if Shodan_Item_Host:
if 'port' in Shodan_Item_Host:
if int(Shodan_Item['port']) not in [
80, 443
]:
Shodan_Item_Port = Shodan_Item['port']
if Shodan_Item_Port != 0:
Shodan_Item_URL = f"{Shodan_Item_Module}://{Shodan_Item_Host}:{str(Shodan_Item_Port)}"
else:
Shodan_Item_URL = f"{Shodan_Item_Module}://{Shodan_Item_Host}"
Title = "Shodan | " + str(Shodan_Item_Host)
if Shodan_Item_URL not in Cached_Data and Shodan_Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Local_Plugin_Name,
Shodan_Item_Response, Shodan_Item_Host,
The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output(
[Main_File, Output_file],
Shodan_Item_URL, Title,
Plugin_Name.lower())
Data_to_Cache.append(Shodan_Item_URL)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
Current_Step += 1
elif Type == "Host":
Local_Plugin_Name = Plugin_Name + "-Host"
try:
API_Response = API_Session.host(Query)
except Exception as e:
logging.error(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}."
)
break
JSON_Object = Common.JSON_Handler(API_Response)
JSON_Output_Response = JSON_Object.Dump_JSON()
Main_File = General.Main_File_Create(
Directory, Local_Plugin_Name, JSON_Output_Response,
Query, The_File_Extensions["Main"])
Output_Connections = General.Connections(
Query, Local_Plugin_Name, Domain, "Domain Information",
Task_ID, Plugin_Name.lower())
Shodan_URL = f"https://www.{Domain}/host/{Query}"
Title = "Shodan | " + Query
if Shodan_URL not in Cached_Data and Shodan_URL not in Data_to_Cache:
Shodan_Responses = Common.Request_Handler(
Shodan_URL,
Application_JSON_CT=True,
Accept_XML=True,
Accept_Language_EN_US=True,
Filter=True,
Host=f"https://www.{Domain}")
Shodan_Response = Shodan_Responses["Filtered"]
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Shodan_Response,
Query, The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output([Main_File, Output_file],
Shodan_URL, Title,
Plugin_Name.lower())
Data_to_Cache.append(Shodan_URL)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - No results found."
)
except:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to complete task."
)
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(self):
try:
Data_to_Cache = []
Directory = General.Make_Directory(self.Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, self.Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File),
"w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data_Object = General.Cache(Directory, self.Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
for Query in self.Query_List:
try:
if self.Type == "ABN":
Main_URL = f'https://{self.Domain}/ABN/View?id=' + Query
Responses = Common.Request_Handler(
Main_URL,
Filter=True,
Host=f"https://www.{self.Domain}")
Response = Responses["Regular"]
try:
if 'Error searching ABN Lookup' not in Response:
Query = str(int(Query))
Response = Responses["Filtered"]
if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, self.Plugin_Name,
Response, General.Get_Title(Main_URL),
self.The_File_Extensions["Query"])
if Output_file:
Output_Connections = General.Connections(
Query, self.Plugin_Name,
self.Domain, self.Result_Type,
self.Task_ID, self.Plugin_Name)
Output_Connections.Output(
[Output_file], Main_URL,
General.Get_Title(Main_URL).strip(
" | ABN Lookup"),
self.Concat_Plugin_Name)
Data_to_Cache.append(Main_URL)
else:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist."
)
else:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - ABN Lookup returned error."
)
except:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid query provided for ABN Search."
)
elif self.Type == "ACN":
Main_URL = f'https://{self.Domain}/Search/Run'
Data = {
'SearchParameters.SearchText':
Query,
'SearchParameters.AllNames':
'true',
'ctl00%24ContentPagePlaceholder%24SearchBox%24MainSearchButton':
'Search'
}
Responses = Common.Request_Handler(
Main_URL,
Method="POST",
Filter=True,
Host=f"https://www.{self.Domain}",
Data=Data)
Response = Responses["Regular"]
Filtered_Response = Responses["Filtered"]
try:
ACN_Regex = Common.Regex_Handler(
Query, Type="Company_Name")
if ACN_Regex:
Main_File = General.Main_File_Create(
Directory, self.Plugin_Name,
Filtered_Response, Query,
self.The_File_Extensions["Main"])
Current_Step = 0
ABNs_Regex = Common.Regex_Handler(
Response,
Custom_Regex=
r"\<input\sid\=\"Results\_NameItems\_\d+\_\_Compressed\"\sname\=\"Results\.NameItems\[\d+\]\.Compressed\"\stype\=\"hidden\"\svalue\=\"(\d{11})\,\d{2}\s\d{3}\s\d{3}\s\d{3}\,0000000001\,Active\,active\,([\d\w\s\&\-\_\.]+)\,Current\,",
Findall=True)
if ABNs_Regex:
Output_Connections = General.Connections(
Query, self.Plugin_Name, self.Domain,
self.Result_Type, self.Task_ID,
self.Plugin_Name)
for ABN_URL, ACN in ABNs_Regex:
Full_ABN_URL = f'https://{self.Domain}/ABN/View?abn={ABN_URL}'
if Full_ABN_URL not in Cached_Data and Full_ABN_URL not in Data_to_Cache and Current_Step < int(
self.Limit):
ACN = ACN.rstrip()
Current_Responses = Common.Request_Handler(
Full_ABN_URL,
Filter=True,
Host=
f"https://www.{self.Domain}")
Current_Response = Current_Responses[
"Filtered"]
Output_file = General.Create_Query_Results_Output_File(
Directory, Query,
self.Plugin_Name,
str(Current_Response),
ACN.replace(' ', '-'), self.
The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output(
[Main_File, Output_file],
Full_ABN_URL,
General.Get_Title(
Full_ABN_URL).strip(
" | ABN Lookup"),
self.Concat_Plugin_Name)
Data_to_Cache.append(
Full_ABN_URL)
else:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist."
)
Current_Step += 1
else:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Response did not match regular expression."
)
else:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Query did not match regular expression."
)
except:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid query provided for ACN Search."
)
else:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Invalid request type."
)
except:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to make request."
)
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(
f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(self):
try:
Data_to_Cache = []
Directory = General.Make_Directory(self.Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, self.Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data_Object = General.Cache(Directory, self.Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
for Query in self.Query_List:
URL_Query = urllib.parse.quote(Query)
URL = f"https://api.duckduckgo.com/?q={URL_Query}&format=json"
DDG_Response = Common.Request_Handler(URL)
JSON_Object = Common.JSON_Handler(DDG_Response)
JSON_Response = JSON_Object.To_JSON_Loads()
JSON_Output_Response = JSON_Object.Dump_JSON()
Main_File = General.Main_File_Create(Directory, self.Plugin_Name, JSON_Output_Response, Query, self.The_File_Extensions["Main"])
Output_Connections = General.Connections(Query, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower())
if JSON_Response.get('RelatedTopics'):
Current_Step = 0
for DDG_Item_Link in JSON_Response['RelatedTopics']:
try:
if 'FirstURL' in DDG_Item_Link:
DDG_URL = DDG_Item_Link['FirstURL']
Title = General.Get_Title(DDG_URL)
Title = f"DuckDuckGo | {Title}"
if DDG_URL not in Cached_Data and DDG_URL not in Data_to_Cache and Current_Step < int(self.Limit):
DDG_Item_Responses = Common.Request_Handler(DDG_URL, Filter=True, Host=f"https://www.{self.Domain}")
DDG_Item_Response = DDG_Item_Responses["Filtered"]
Output_file = General.Create_Query_Results_Output_File(Directory, Query, self.Plugin_Name, DDG_Item_Response, DDG_URL, self.The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output([Main_File, Output_file], DDG_URL, Title, self.Plugin_Name.lower())
Data_to_Cache.append(DDG_URL)
else:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to create output file. File may already exist.")
Current_Step += 1
else:
break
elif 'Topics' in DDG_Item_Link:
if type(DDG_Item_Link['Topics']) == list:
JSON_Response['RelatedTopics'].extend(DDG_Item_Link['Topics'])
except Exception as e:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
else:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - No results found.")
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs):
try:
Data_to_Cache = []
Results = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Reddit_Details = Load_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Limit = General.Get_Limit(kwargs)
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
Reddit_Connection = praw.Reddit(
client_id=Reddit_Details[0],
client_secret=Reddit_Details[1],
user_agent=Reddit_Details[2],
username=Reddit_Details[3],
password=Reddit_Details[4])
All_Subreddits = Reddit_Connection.subreddit(Reddit_Details[5])
for Subreddit in All_Subreddits.search(
Query, limit=Limit
): # Limit, subreddit and search to be controlled by the web app.
Results.append(Subreddit.url)
except:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to get results. Are you connected to the internet?"
)
Output_Connections = General.Connections(Query, Plugin_Name,
"reddit.com", "Forum",
Task_ID,
Plugin_Name.lower())
for Result in Results:
if Result not in Cached_Data and Result not in Data_to_Cache:
try:
Reddit_Regex = re.search(
"https\:\/\/www\.reddit\.com\/r\/(\w+)\/comments\/(\w+)\/([\w\d]+)\/",
Result[0])
if Reddit_Regex:
Reddit_Response = requests.get(
Result, headers=headers).text
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Reddit_Response,
Reddit_Regex.group(3), The_File_Extension)
if Output_file:
Output_Connections.Output(
[Output_file], Result,
General.Get_Title(Result[0]),
Plugin_Name.lower())
Data_to_Cache.append(Result[0])
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
except:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create file."
)
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Limit=10):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Craigslist_Location = Load_Configuration()
Cached_Data_Object = General.Cache(Directory, Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(Limit)
for Query in Query_List:
if Craigslist_Location:
Main_URL = f"https://{Craigslist_Location.lower()}.craigslist.org/search/sss?format=rss&query={Query}"
Craigslist_Response = feedparser.parse(Main_URL)
Craigslist_Items = Craigslist_Response["items"]
Current_Step = 0
for Item in Craigslist_Items:
Item_URL = Item["link"]
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Craigslist_Responses = Common.Request_Handler(
Item_URL,
Filter=True,
Host=
f"https://{Craigslist_Location.lower()}.craigslist.org"
)
Craigslist_Response = Craigslist_Responses["Filtered"]
Local_URL = f"https://{Craigslist_Location.lower()}.craigslist.org/"
Local_Domain = f"{Craigslist_Location.lower()}.craigslist.org"
Filename = Item_URL.replace(Local_URL, "")
Filename = Filename.replace(".html/", "")
Filename = Filename.replace(".html", "")
Filename = Filename.replace("/", "-")
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Craigslist_Response,
Filename, The_File_Extension)
if Output_file:
Output_Connections = General.Connections(
Query, Plugin_Name, Local_Domain,
"Search Result", Task_ID, Plugin_Name.lower())
Output_Connections.Output(
[Output_file], Item_URL,
General.Get_Title(Item_URL),
Plugin_Name.lower())
Data_to_Cache.append(Item_URL)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
Current_Step += 1
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Yandex_Details = Load_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(kwargs)
for Query in Query_List:
Yandex_Response = General.Request_Handler(
f"https://{Domain}/search/xml?user={Yandex_Details[0]}&key={Yandex_Details[1]}&query={Query}&l10n=en&sortby=rlv&filter=none&maxpassages=five&groupby=attr% 3D% 22% 22.mode% 3Dflat.groups-on-page% 3D{str(Limit)}.docs-in-group% 3D1"
)
JSON_Response = xmltodict.parse(Yandex_Response)
JSON_Output_Response = json.dumps(JSON_Response,
indent=4,
sort_keys=True)
Main_File = General.Main_File_Create(Directory, Plugin_Name,
JSON_Output_Response, Query,
The_File_Extensions["Main"])
Output_Connections = General.Connections(Query, Plugin_Name,
Domain, "Search Result",
Task_ID,
Plugin_Name.lower())
New_JSON_Response = Recursive_Dict_Check(
["yandexsearch", "response", "results", "grouping", "group"],
JSON_Response)
if New_JSON_Response:
for Yandex_Item_Line in New_JSON_Response:
try:
if Recursive_Dict_Check(["doc", "url"],
Yandex_Item_Line):
Yandex_Item_Line = Yandex_Item_Line['doc']
Yandex_URL = Yandex_Item_Line['url']
Title = Recursive_Dict_Check(["title", "#text"],
JSON_Response)
if Title:
Title = f"Yandex | {Title}"
else:
Title = General.Get_Title(Yandex_URL)
Title = f"Yandex | {Title}"
if Yandex_URL not in Cached_Data and Yandex_URL not in Data_to_Cache:
Yandex_Item_Responses = General.Request_Handler(
Yandex_URL,
Application_JSON_CT=True,
Accept_XML=True,
Accept_Language_EN_US=True,
Filter=True,
Host=f"https://{Domain}")
Yandex_Item_Response = Yandex_Item_Responses[
"Filtered"]
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name,
Yandex_Item_Response, Yandex_URL,
The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output(
[Main_File, Output_file], Yandex_URL,
Title, Plugin_Name.lower())
Data_to_Cache.append(Yandex_URL)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}"
)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - No results found."
)
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Type, Limit=10):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data_Object = General.Cache(Directory, Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
if Type == "UKBN":
Authorization_Key = Load_Configuration()
if Authorization_Key:
Authorization_Key = "Basic " + Authorization_Key.decode('ascii')
headers_auth = {"Authorization": Authorization_Key}
Main_URL = f'https://api.{Domain}/company/{Query}'
Response = Common.Request_Handler(Main_URL, Optional_Headers=headers_auth)
JSON_Object = Common.JSON_Handler(Response)
JSON_Response = JSON_Object.To_JSON_Loads()
Indented_JSON_Response = JSON_Object.Dump_JSON()
try:
Query = str(int(Query))
if Response and '{"errors":[{"error":"company-profile-not-found","type":"ch:service"}]}' not in Response:
if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache:
Current_Company_Number = str(JSON_Response["company_number"])
Result_URL = f'https://beta.{Domain}/company/{Current_Company_Number}'
Result_Responses = Common.Request_Handler(Result_URL, Filter=True, Host=f"https://beta.{Domain}")
Result_Response = Result_Responses["Filtered"]
UKCN = str(JSON_Response["company_name"])
Main_Output_File = General.Main_File_Create(Directory, Plugin_Name, Indented_JSON_Response, Query, The_File_Extensions["Main"])
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Result_Response, UKCN, The_File_Extensions["Query"])
if Output_file:
Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Company Details", Task_ID, Plugin_Name)
Output_Connections.Output([Main_Output_File, Output_file], Result_URL, f"UK Business Number {Query}", Concat_Plugin_Name)
Data_to_Cache.append(Main_URL)
else:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
except:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid query provided for UKBN Search.")
else:
logging.info(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to retrieve API key.")
elif Type == "UKCN":
Authorization_Key = Load_Configuration()
if Authorization_Key:
Authorization_Key = "Basic " + Authorization_Key.decode('ascii')
Limit = General.Get_Limit(Limit)
try:
Main_URL = f'https://api.{Domain}/search/companies?q={Query}&items_per_page={Limit}'
headers_auth = {"Authorization": Authorization_Key}
Response = Common.Request_Handler(Main_URL, Optional_Headers=headers_auth)
JSON_Object = Common.JSON_Handler(Response)
JSON_Response = JSON_Object.To_JSON_Loads()
Indented_JSON_Response = JSON_Object.Dump_JSON()
try:
if JSON_Response['total_results'] > 0:
Main_Output_File = General.Main_File_Create(Directory, Plugin_Name, Indented_JSON_Response, Query, The_File_Extensions["Main"])
Output_Connections = General.Connections(Query, Plugin_Name, Domain, "Company Details", Task_ID, Plugin_Name)
for Item in JSON_Response['items']:
UKBN_URL = Item['links']['self']
Full_UKBN_URL = f'https://beta.{Domain}{str(UKBN_URL)}'
UKBN = UKBN_URL.strip("/company/")
if Full_UKBN_URL not in Cached_Data and Full_UKBN_URL not in Data_to_Cache:
UKCN = Item['title']
Current_Responses = Common.Request_Handler(Full_UKBN_URL, Filter=True, Host=f"https://beta.{Domain}")
Current_Response = Current_Responses["Filtered"]
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, str(Current_Response), UKCN, The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output([Main_Output_File, Output_file], Full_UKBN_URL, f"UK Business Number {UKBN} for Query {Query}", Concat_Plugin_Name)
Data_to_Cache.append(Full_UKBN_URL)
else:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist.")
except:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Error during UKCN Search, perhaps the rate limit has been exceeded.")
except:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid query provided for UKCN Search.")
else:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to retrieve API key.")
else:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Invalid request type.")
except:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to make request.")
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data_Object = General.Cache(Directory, Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
Main_URL = f"http://{Domain}/{Query}"
Responses = Common.Request_Handler(Main_URL,
Filter=True,
Host=f"https://www.{Domain}")
Response = Responses["Regular"]
Filtered_Response = Responses["Filtered"]
Kik_Item_Regex = Common.Regex_Handler(
Response,
Custom_Regex=
rf"\<h1\sclass\=\"display\-name\"\>(.+)\<\/h1>\s+\<h2\sclass\=\"username\"\>{Query}\<\/h2\>"
)
if Kik_Item_Regex:
if Kik_Item_Regex.group(1) != " ":
Output_Connections = General.Connections(
Query, Plugin_Name, Domain, "Social Media - Person",
Task_ID, Plugin_Name.lower())
Title = f"Kik | {Kik_Item_Regex.group(1)}"
if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache:
Output_file = General.Main_File_Create(
Directory, Plugin_Name, Filtered_Response, Query,
The_File_Extension)
if Output_file:
print(Main_URL, Title)
Output_Connections.Output([Output_file],
Main_URL, Title,
Plugin_Name.lower())
Data_to_Cache.append(Main_URL)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
else:
logging.info(
f"{Common.Date()} - {__name__.strip('plugins.')} - Query didn't match regex pattern."
)
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, Limit=10):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data_Object = General.Cache(Directory, Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(Limit)
for Query in Query_List:
# Query can be Title or ISBN
Main_URL = f"http://{Domain}/search.php?req={Query}&lg_topic=libgen&open=0&view=simple&res=100&phrase=1&column=def"
Lib_Gen_Response = Common.Request_Handler(Main_URL)
Main_File = General.Main_File_Create(Directory, Plugin_Name,
Lib_Gen_Response, Query,
The_File_Extension)
Lib_Gen_Regex = Common.Regex_Handler(
Lib_Gen_Response,
Custom_Regex=r"book\/index\.php\?md5=[A-Fa-f0-9]{32}",
Findall=True)
if Lib_Gen_Regex:
Current_Step = 0
for Regex in Lib_Gen_Regex:
Item_URL = f"http://{Domain}/{Regex}"
Title = General.Get_Title(Item_URL).replace(
"Genesis:", "Genesis |")
Lib_Item_Responses = Common.Request_Handler(
Item_URL, Filter=True, Host=f"http://{Domain}")
Lib_Item_Response = Lib_Item_Responses["Filtered"]
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Lib_Item_Response,
Regex, The_File_Extension)
if Output_file:
Output_Connections = General.Connections(
Query, Plugin_Name, Domain, "Publication",
Task_ID, Concat_Plugin_Name)
Output_Connections.Output([Main_File, Output_file],
Item_URL, Title,
Concat_Plugin_Name)
Data_to_Cache.append(Item_URL)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
Current_Step += 1
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression."
)
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
try:
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/RSS_Feeds.txt')
Current_File = open(Configuration_File, "r") # Open the provided file and retrieve each client to test.
URLs = Current_File.read().splitlines()
Current_File.close()
except:
logging.warning(General.Date() + " Please provide a valid file, failed to open the file which contains the data to search for.")
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
for URL in URLs: # URLs to be controlled by the web app.
RSS = feedparser.parse(URL)
Current_Step = 0
for Feed in RSS.entries:
if Query in Feed.description:
Dump_Types = General.Data_Type_Discovery(Feed.description)
File_Link = Feed.link.replace("https://", "")
File_Link = File_Link.replace("http://", "")
File_Link = File_Link.replace("www.", "")
File_Link = File_Link.replace("/", "-")
Domain = URL.replace("https://", "")
Domain = Domain.replace("http://", "")
Domain = Domain.replace("www.", "")
if Feed.link not in Cached_Data and Feed.link not in Data_to_Cache and Current_Step < int(Limit):
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Feed.description, File_Link, The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name, Feed.link, Domain, "Data Leakage", Task_ID, General.Get_Title(Feed.link), Plugin_Name.lower(), Dump_Types=Dump_Types)
Data_to_Cache.append(Feed.link)
Current_Step += 1
else:
logging.info(General.Date() + " Query not found.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Limit=10):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
YouTube_Details = Load_Configuration()
Cached_Data_Object = General.Cache(Directory, Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(Limit)
for Query in Query_List:
YouTube_Handler = discovery.build(YouTube_Details[1],
YouTube_Details[2],
developerKey=YouTube_Details[0],
cache_discovery=False)
Search_Response = YouTube_Handler.search().list(
q=Query,
type='video',
part='id,snippet',
maxResults=Limit,
).execute()
JSON_Object = Common.JSON_Handler(Search_Response.get('items', []))
JSON_Output_Response = JSON_Object.Dump_JSON()
Main_File = General.Main_File_Create(Directory, Plugin_Name,
JSON_Output_Response, Query,
The_File_Extensions["Main"])
Output_Connections = General.Connections(Query, Plugin_Name,
Domain,
"Social Media - Media",
Task_ID,
Plugin_Name.lower())
for Search_Result in Search_Response.get('items', []):
Full_Video_URL = f"https://www.{Domain}/watch?v=" + Search_Result[
'id']['videoId']
Search_Video_Responses = Common.Request_Handler(
Full_Video_URL, Filter=True, Host=f"https://www.{Domain}")
Search_Video_Response = Search_Video_Responses["Filtered"]
Title = "YouTube | " + Search_Result['snippet']['title']
if Full_Video_URL not in Cached_Data and Full_Video_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Search_Video_Response,
Search_Result['id']['videoId'],
The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output([Main_File, Output_file],
Full_Video_URL, Title,
Plugin_Name.lower())
Data_to_Cache.append(Full_Video_URL)
else:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
