def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir,
'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') +
" - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for Reddit_Details in Configuration_Data[Plugin_Name.lower()]:
Reddit_Client_ID = Reddit_Details['client_id']
Reddit_Client_Secret = Reddit_Details['client_secret']
Reddit_User_Agent = Reddit_Details['user_agent']
Reddit_Username = Reddit_Details['username']
Reddit_Password = Reddit_Details['password']
Subreddit_to_Search = Reddit_Details["subreddits"]
if Reddit_Client_ID and Reddit_Client_Secret and Reddit_User_Agent and Reddit_Username and Reddit_Password and Subreddit_to_Search:
return [
Reddit_Client_ID, Reddit_Client_Secret,
Reddit_User_Agent, Reddit_Username, Reddit_Password,
Subreddit_to_Search
]
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') +
" - Failed to load Reddit details.")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir,
'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') +
" - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for Flickr_Details in Configuration_Data[Plugin_Name.lower()]:
if Flickr_Details['api_key'] and Flickr_Details['api_secret']:
return [
Flickr_Details['api_key'], Flickr_Details['api_secret']
]
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') +
" - Failed to load location details.")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for YouTube_Details in Configuration_Data[Plugin_Name.lower()]:
YouTube_Developer_Key = YouTube_Details['developer_key']
YouTube_Application_Name = YouTube_Details['application_name']
YouTube_Application_Version = YouTube_Details['application_version']
YouTube_Location = YouTube_Details['location']
YouTube_Location_Radius = YouTube_Details['location_radius']
if YouTube_Developer_Key and YouTube_Application_Name and YouTube_Application_Version and YouTube_Location and YouTube_Location_Radius:
return [YouTube_Developer_Key, YouTube_Application_Name, YouTube_Application_Version, YouTube_Location, YouTube_Location_Radius]
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to load location details.")
def General_Pull(Handle, Limit, Directory, API, Task_ID):
Data_to_Cache = []
Cached_Data = []
JSON_Response = []
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Latest_Tweets = API.user_timeline(screen_name=Handle, count=Limit)
for Tweet in Latest_Tweets:
Link = ""
try:
JSON_Response.append({
'id': Tweet.id,
'text': Tweet.text,
'author_name': Tweet.user.screen_name,
'url': Tweet.entities['urls'][0]["expanded_url"]
})
Link = Tweet.entities['urls'][0]["expanded_url"]
except:
JSON_Response.append({
'id': Tweet.id,
'text': Tweet.text,
'author_name': Tweet.user.screen_name
})
JSON_Output = json.dumps(JSON_Response, indent=4, sort_keys=True)
for JSON_Item in JSON_Response:
if 'text' in JSON_Item and 'url' in JSON_Item:
Link = JSON_Item['url']
if Link not in Cached_Data and Link not in Data_to_Cache:
logging.info(General.Date() + " " + Link)
Item_Response = requests.get(Link).text
Output_file = General.Create_Query_Results_Output_File(Directory, Handle, Plugin_Name, Item_Response, str(JSON_Item['id']), ".html")
if Output_file:
General.Connections(Output_file, Handle, Plugin_Name, Link, "twitter.com", "Data Leakage", Task_ID, General.Get_Title(Link), Plugin_Name.lower())
else:
logging.warning(General.Date() + " Output file not returned.")
else:
logging.warning(General.Date() + " Insufficient parameters provided.")
Data_to_Cache.append(Link)
General.Main_File_Create(Directory, Plugin_Name, JSON_Output, Handle, ".json")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Twitter_Credentials = Load_Configuration()
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
Authentication = tweepy.OAuthHandler(Twitter_Credentials[0], Twitter_Credentials[1])
Authentication.set_access_token(Twitter_Credentials[2], Twitter_Credentials[3])
API = tweepy.API(Authentication)
General_Pull(Query, Limit, Directory, API, Task_ID)
except:
logging.info(General.Date() + " Failed to get results. Are you connected to the internet?")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir,
'plugins/common/config/config.json')
logging.info(General.Date() + " Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for Vulners_Details in Configuration_Data[Plugin_Name.lower()]:
if Vulners_Details['api_key']:
return Vulners_Details
else:
return None
except:
logging.warning(General.Date() + " Failed to load location details.")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/config.json')
logging.info(General.Date() + " Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for HIBP_Details in Configuration_Data[Concat_Plugin_Name]:
API_Key = HIBP_Details['api_key']
if API_Key:
return API_Key
else:
return None
except:
logging.warning(General.Date() + " Failed to load API details.")
def Search(Query_List, Task_ID):
Data_to_Cache = []
Cached_Data = []
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
BSB_Search_URL = "https://www.bsbnumbers.com/" + Query + ".html"
Response = requests.get(BSB_Search_URL).text
Error_Regex = re.search(r"Correct\sthe\sfollowing\serrors", Response)
if not Error_Regex:
if BSB_Search_URL not in Cached_Data and BSB_Search_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Response, Query,
The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name,
BSB_Search_URL, "bsbnumbers.com",
"Data Leakage", Task_ID,
General.Get_Title(BSB_Search_URL),
Plugin_Name.lower())
Data_to_Cache.append(BSB_Search_URL)
else:
logging.warning(General.Date() +
" Query returned error, probably does not exist.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for API_Details in Configuration_Data[Concat_Plugin_Name]:
API_Key = API_Details['api_key']
if API_Key:
API_Key = base64.b64encode(API_Key.encode('ascii'))
return API_Key
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to load location details.")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/config.json')
logging.info(General.Date() + " Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for Twitter_Details in Configuration_Data[Plugin_Name.lower()]:
Consumer_Key = Twitter_Details['CONSUMER_KEY']
Consumer_Secret = Twitter_Details['CONSUMER_SECRET']
Access_Key = Twitter_Details['ACCESS_KEY']
Access_Secret = Twitter_Details['ACCESS_SECRET']
if Consumer_Key and Consumer_Secret and Access_Key and Access_Secret:
return [Consumer_Key, Consumer_Secret, Access_Key, Access_Secret]
else:
return None
except:
logging.warning(General.Date() + " Failed to load Twitter details.")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for Google_Details in Configuration_Data[Plugin_Name.lower()]:
Google_CX = Google_Details['cx']
Google_Developer_Key = Google_Details['developer_key']
Google_Application_Name = Google_Details['application_name']
Google_Application_Version = Google_Details['application_version']
if Google_CX and Google_Developer_Key and Google_Application_Name and Google_Application_Version:
return [Google_CX, Google_Developer_Key, Google_Application_Name, Google_Application_Version]
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to load API details.")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/config.json')
logging.info(General.Date() + "[+] Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
Craigslist_Details = Configuration_Data[Plugin_Name.lower()]
if Craigslist_Details['city']:
return Craigslist_Details['city']
else:
return None
except:
logging.warning(f"{General.Date()} - {__name__.strip('plugins.')} - Failed to load location details.")
def Load_Configuration():
logging.info(General.Date() + "[+] Loading configuration data.")
try:
with open(Connectors.Set_Configuration_File()) as JSON_File:
Configuration_Data = json.load(JSON_File)
Craigslist_Details = Configuration_Data[Plugin_Name.lower()]
if Craigslist_Details['city']:
return Craigslist_Details['city']
else:
return None
except:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to load location details."
)
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
else:
Limit = 10
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
# Query can be Title or ISBN
Main_URL = "http://gen.lib.rus.ec/search.php?req=" + Query + "&lg_topic=libgen&open=0&view=simple&res=100&phrase=1&column=def"
Lib_Gen_Response = requests.get(Main_URL).text
General.Main_File_Create(Directory, Plugin_Name, Lib_Gen_Response,
Query, The_File_Extension)
Lib_Gen_Regex = re.findall("book\/index\.php\?md5=[A-Fa-f0-9]{32}",
Lib_Gen_Response)
if Lib_Gen_Regex:
Current_Step = 0
for Regex in Lib_Gen_Regex:
Item_URL = "http://gen.lib.rus.ec/" + Regex
Lib_Item_Response = requests.get(Item_URL).text
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Lib_Item_Response,
Regex, The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name,
Item_URL, "gen.lib.rus.ec",
"Data Leakage", Task_ID,
General.Get_Title(Item_URL),
Concat_Plugin_Name)
Data_to_Cache.append(Item_URL)
Current_Step += 1
else:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') +
" - Failed to match regular expression.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
Results = []
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Reddit_Details = Load_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
Reddit_Connection = praw.Reddit(client_id=Reddit_Details[0], \
client_secret=Reddit_Details[1], \
user_agent=Reddit_Details[2], \
username=Reddit_Details[3], \
password=Reddit_Details[4])
All_Subreddits = Reddit_Connection.subreddit(Reddit_Details[5])
for Subreddit in All_Subreddits.search(Query, limit=Limit): # Limit, subreddit and search to be controlled by the web app.
Current_Result = []
Current_Result.append(Subreddit.url)
Current_Result.append(Subreddit.selftext)
Results.append(Current_Result)
except:
logging.warning(General.Date() + " Failed to get results. Are you connected to the internet?")
for Result in Results:
if Result[0] not in Cached_Data and Result[0] not in Data_to_Cache:
try:
Reddit_Regex = re.search("https\:\/\/www\.reddit\.com\/r\/(\w+)\/comments\/(\w+)\/([\w\d]+)\/", Result[0])
if Reddit_Regex:
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Result[1], Reddit_Regex.group(3), The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name, Result[0], "reddit.com", "Data Leakage", Task_ID, General.Get_Title(Result[0]), Plugin_Name.lower())
except:
logging.warning(General.Date() + " Failed to create file.")
Data_to_Cache.append(Result[0])
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Google_Details = Load_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
Service = build("customsearch", Google_Details[3], developerKey=Google_Details[1])
CSE_Response = Service.cse().list(q=Query, cx=Google_Details[0], num=Limit).execute()
CSE_JSON_Output_Response = json.dumps(CSE_Response, indent=4, sort_keys=True)
CSE_JSON_Response = json.loads(CSE_JSON_Output_Response)
General.Main_File_Create(Directory, Plugin_Name, CSE_JSON_Output_Response, Query, ".json")
Output_Connections = General.Connections(Query, Plugin_Name, "google.com", "Domain Spoof", Task_ID, Plugin_Name.lower())
for JSON_Response_Items in CSE_JSON_Response['items']:
try:
Google_Item = JSON_Response_Items['pagemap']['metatags']
for Google_Item_Line in Google_Item:
Google_Item_URL = Google_Item_Line['og:url']
if Google_Item_URL not in Cached_Data and Google_Item_URL not in Data_to_Cache:
Path_Regex = re.search(r"https?\:\/\/(www\.)?[\w\d\.]+\.\w{2,3}(\.\w{2,3})?(\.\w{2,3})?\/([\w\d\-\_\/]+)?", Google_Item_URL)
if Path_Regex:
headers = {'Content-Type': 'application/json', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0', 'Accept': 'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5'}
Google_Item_Response = requests.get(Google_Item_URL, headers=headers).text
Output_Path = Path_Regex.group(4).replace("/", "-")
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Google_Item_Response, Output_Path, The_File_Extension)
if Output_file:
Output_Connections.Output(Output_file, Google_Item_URL, General.Get_Title(Google_Item_URL))
else:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to match regular expression.")
Data_to_Cache.append(Google_Item_URL)
except Exception as e:
logging.info(General.Date() + str(e))
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID):
Data_to_Cache = []
Cached_Data = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
Output_Connections = General.Connections(
Query, Plugin_Name, "general-insurance.coles.com.au",
"Data Leakage", Task_ID, Concat_Plugin_Name)
for State in States:
Post_URL = 'https://general-insurance.coles.com.au/bin/wesfarmers/search/vehiclerego'
data = '''{"isRegoSearch":"YES","regoSearchCount":2,"regoMatchCount":1,"regoSearchFailureCount":0,"failPaymentAttempts":0,"pauseStep":"false","campaignBaseURL":"https://secure.colesinsurance.com.au/campaignimages/","sessionState":"OPEN","sessionStep":"0","policyHolders":[],"updateSessionURL":"http://dev.gtw.gp-mdl.auiag.corp:9000/sys/colessessionservice/motor/v1/update-session","insuranceType":"COMP","startDate":"03/07/2019","drivers":[{"driverRef":"MainDriver","yearsLicenced":{"vehRef":"veh1"}}],"priceBeatAttemptsRemaining":"2","currentInsurerOptions":[{"id":"AAMI","value":"AAMI","text":"AAMI"},{"id":"Allianz","value":"Allianz","text":"Allianz"},{"id":"Apia","value":"Apia","text":"Apia"},{"id":"Bingle","value":"Bingle","text":"Bingle"},{"id":"Broker","value":"Broker","text":"Broker"},{"id":"BudgDirect","value":"BudgDirect","text":"Budget Direct"},{"id":"Buzz","value":"Buzz","text":"Buzz"},{"id":"CGU","value":"CGU","text":"CGU"},{"id":"Coles","value":"Coles","text":"Coles"},{"id":"CommInsure","value":"CommInsure","text":"CommInsure"},{"id":"GIO","value":"GIO","text":"GIO"},{"id":"HBF","value":"HBF","text":"HBF"},{"id":"JustCar","value":"JustCar","text":"Just Car"},{"id":"NRMA","value":"NRMA","text":"NRMA"},{"id":"Progress","value":"Progress","text":"Progressive"},{"id":"QBE","value":"QBE","text":"QBE"},{"id":"RAA","value":"RAA","text":"RAA"},{"id":"RAC","value":"RAC","text":"RAC"},{"id":"RACQ","value":"RACQ","text":"RACQ"},{"id":"RACT","value":"RACT","text":"RACT"},{"id":"RACV","value":"RACV","text":"RACV"},{"id":"Real","value":"Real","text":"Real"},{"id":"SGIC","value":"SGIC","text":"SGIC"},{"id":"SGIO","value":"SGIO","text":"SGIO"},{"id":"Shannons","value":"Shannons","text":"Shannons"},{"id":"Suncorp","value":"Suncorp","text":"Suncorp"},{"id":"Youi","value":"Youi","text":"Youi"},{"id":"None","value":"None","text":"Car is not currently insured"},{"id":"Dontknow","value":"Dontknow","text":"Don't Know"},{"id":"Other","value":"Other","text":"Other"}],"coverLevelOptions":[{"id":"Gold","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"NRMA","code":"Gold","order":"1"},{"id":"Gold1","value":"Gold Comprehensive Car Insurance","text":"Gold Comprehensive Car Insurance","flagname":"BudgDirect","code":"Gold","order":"1"},{"id":"Standard2","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"SGIC","code":"Standard","order":"2"},{"id":"Gold6","value":"Comprehensive Advantages Car Insurance","text":"Comprehensive Advantages Car Insurance","flagname":"Suncorp","code":"Gold","order":"1"},{"id":"Standard","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"GIO","code":"Standard","order":"2"},{"id":"Standard0","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"NRMA","code":"Standard","order":"2"},{"id":"Gold4","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"SGIC","code":"Gold","order":"1"},{"id":"Standard5","value":"Full Comprehensive Car Insurance","text":"Full Comprehensive Car Insurance","flagname":"1300 Insurance","code":"Standard","order":"2"},{"id":"Gold5","value":"Comprehensive Plus Car Insurance","text":"Comprehensive Plus Car Insurance","flagname":"SGIO","code":"Gold","order":"1"},{"id":"Gold2","value":"Platinum Car Insurance","text":"Platinum Car Insurance","flagname":"GIO","code":"Gold","order":"1"},{"id":"Standard3","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"SGIO","code":"Standard","order":"2"},{"id":"Gold3","value":"Complete Care Motor Insurance","text":"Complete Care Motor Insurance","flagname":"RACV","code":"Gold","order":"1"},{"id":"Standard4","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"Suncorp","code":"Standard","order":"2"},{"id":"Gold0","value":"Gold Comprehensive Car Insurance","text":"Gold Comprehensive Car Insurance","flagname":"1300 Insurance","code":"Gold","order":"1"},{"id":"Standard1","value":"Comprehensive Car Insurance","text":"Comprehensive Car Insurance","flagname":"RACV","code":"Standard","order":"2"}],"riskAddress":{"latitude":"-33.86579240","locality":"PYRMONT","postcode":"2009","extraAddressInfo":{"houseNumber":"1","lotNumber":"1","streetName":"HARRIS","streetSuffix":"STREET","unitNumber":"1"},"state":"''' + State + '''","line3":null,"isVerificationRequired":null,"gnaf":"GANSW709981139","line2":null,"line1":"1 Harris Street","longitude":"151.19109690","displayString":"1 HARRIS STREET, PYRMONT, NSW, 2009"},"postcode":{"latitude":"-33.86579240","locality":"PYRMONT","postcode":"2009","extraAddressInfo":{"houseNumber":"1","lotNumber":"1","streetName":"HARRIS","streetSuffix":"STREET","unitNumber":"1"},"state":"''' + State + '''","line3":null,"isVerificationRequired":null,"gnaf":"GANSW709981139","line2":null,"line1":"1 Harris Street","longitude":"151.19109690","displayString":"1 HARRIS STREET, PYRMONT, NSW, 2009"},"carRegistration":"''' + Query + '''","chooseValue":"","whatValueInsure":"Marketvalue","whatValueInsure_value":{"key":"Marketvalue","value":"Market Value"}}'''
headers = {
'Content-Type': 'ext/plain;charset=UTF-8',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Referer':
'https://general-insurance.coles.com.au/motor/get-quote',
'Origin': 'https://general-insurance.coles.com.au',
'Host': 'general-insurance.coles.com.au'
}
Registration_Response = requests.post(Post_URL,
data=data,
headers=headers).text
Registration_Response = json.loads(Registration_Response)
try:
Title = Registration_Response['vehicles'][0][
'make'] + " " + Registration_Response['vehicles'][0][
'model']
Item_URL = Post_URL + "?" + Query
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name,
json.dumps(Registration_Response,
indent=4,
sort_keys=True), Title.replace(" ", "-"),
The_File_Extension)
if Output_file:
Output_Connections.Output(Output_file, Item_URL, Title)
Data_to_Cache.append(Item_URL)
except:
logging.info(General.Date() + " - " +
__name__.strip('plugins.') +
" - No result found for given query " + Query +
" for state " + State + ".")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, **kwargs):
Data_to_Cache = []
Cached_Data = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
if Type == "UKBN":
Authorization_Key = Load_Configuration()
if Authorization_Key:
Authorization_Key = "Basic " + Authorization_Key.decode('ascii')
headers = {"Authorization": Authorization_Key}
Main_URL = 'https://api.companieshouse.gov.uk/company/' + Query
Response = requests.get(Main_URL, headers=headers).text
JSON_Response = json.loads(Response)
Indented_JSON_Response = json.dumps(JSON_Response, indent=4, sort_keys=True)
try:
Dud_Query = str(int(Query))
if Response and '{"errors":[{"error":"company-profile-not-found","type":"ch:service"}]}' not in Response:
if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache:
Result_URL = 'https://beta.companieshouse.gov.uk/company/' + str(JSON_Response["company_number"])
Result_Response = requests.get(Result_URL).text
Main_Output_File = General.Main_File_Create(Directory, Plugin_Name, Indented_JSON_Response, Query, '.json')
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Result_Response, str(JSON_Response["company_name"]), The_File_Extension)
if Main_Output_File and Output_file:
General.Connections(Output_file, Query, Plugin_Name, Result_URL, "companieshouse.gov.uk", "Data Leakage", Task_ID, str(JSON_Response["company_name"]), Plugin_Name)
Data_to_Cache.append(Main_URL)
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Invalid query provided for UKBN Search.")
else:
logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to retrieve API key.")
elif Type == "UKCN":
Authorization_Key = Load_Configuration()
if Authorization_Key:
Authorization_Key = "Basic " + Authorization_Key.decode('ascii')
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
try:
Main_URL = 'https://api.companieshouse.gov.uk/search/companies?q=' + Query + '&items_per_page=' + Limit
headers = {"Authorization": "Basic SGI5M0V4STRkMDZ2d0NHSzBZTkI5QUxnQ3N3UDNhNEFNMDRHeWtVdzo="}
Response = requests.get(Main_URL, headers=headers).text
JSON_Response = json.loads(Response)
Indented_JSON_Response = json.dumps(JSON_Response, indent=4, sort_keys=True)
try:
if JSON_Response['total_results'] > 0:
General.Main_File_Create(Directory, Plugin_Name, Indented_JSON_Response, Query, '.json')
Output_Connections = General.Connections(Query, Plugin_Name, "companieshouse.gov.uk", "Data Leakage", Task_ID, Plugin_Name)
for Item in JSON_Response['items']:
UKBN_URL = Item['links']['self']
Full_UKBN_URL = 'https://beta.companieshouse.gov.uk' + str(UKBN_URL)
if Full_UKBN_URL not in Cached_Data and Full_UKBN_URL not in Data_to_Cache:
UKCN = Item['title']
Current_Response = requests.get(Full_UKBN_URL).text
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, str(Current_Response), UKCN, The_File_Extension)
if Output_file:
Output_Connections.Output(Output_file, Full_UKBN_URL, UKCN)
Data_to_Cache.append(Full_UKBN_URL)
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Error during UKCN Search, perhaps the rate limit has been exceeded.")
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Invalid query provided for UKCN Search.")
else:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to retrieve API key.")
else:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Invalid request type.")
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to make request.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID):
Data_to_Cache = []
Cached_Data = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
try:
DNS_Info = checkdmarc.check_domains(Query_List)
if len(Query_List) > 1:
for DNS_Item in DNS_Info:
Query = DNS_Item['base_domain']
Output_Dict = json.dumps(DNS_Item, indent=4, sort_keys=True)
Link = "https://www." + Query
Title = "DNS Information for " + DNS_Info['base_domain']
if Link not in Data_to_Cache and Link not in Cached_Data:
Output_file = General.Main_File_Create(
Directory, Plugin_Name, Output_Dict, Query,
The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name,
Link, Query, "Domain Spoof",
Task_ID, Title, Concat_Plugin_Name)
Data_to_Cache.append(Link)
else:
Query = DNS_Info['base_domain']
Output_Dict = json.dumps(DNS_Info, indent=4, sort_keys=True)
Link = "https://www." + Query
Title = "DNS Information for " + Query
if Link not in Data_to_Cache and Link not in Cached_Data:
Output_file = General.Main_File_Create(Directory, Plugin_Name,
Output_Dict, Query,
The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name, Link,
Query, "Domain Spoof", Task_ID, Title,
Concat_Plugin_Name)
Data_to_Cache.append(Link)
except:
logging.warning(General.Date() + " Error retrieving DNS details.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
try:
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(
File_Dir, 'plugins/common/config/RSS_Feeds.txt')
Current_File = open(
Configuration_File,
"r") # Open the provided file and retrieve each client to test.
URLs = Current_File.read().splitlines()
Current_File.close()
except:
logging.warning(
General.Date() + " - " + __name__.strip('plugins.') +
" - Please provide a valid file, failed to open the file which contains the data to search for."
)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
for URL in URLs: # URLs to be controlled by the web app.
RSS = feedparser.parse(URL)
Current_Step = 0
for Feed in RSS.entries:
if Query in Feed.description:
Dump_Types = General.Data_Type_Discovery(Feed.description)
File_Link = Feed.link.replace("https://", "")
File_Link = File_Link.replace("http://", "")
File_Link = File_Link.replace("www.", "")
File_Link = File_Link.replace("/", "-")
Domain = URL.replace("https://", "")
Domain = Domain.replace("http://", "")
Domain = Domain.replace("www.", "")
if Feed.link not in Cached_Data and Feed.link not in Data_to_Cache and Current_Step < int(
Limit):
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Feed.description,
File_Link, The_File_Extension)
if Output_file:
Output_Connections = General.Connections(
Query, Plugin_Name, Domain, "Data Leakage",
Task_ID, Plugin_Name.lower())
Output_Connections.Output(Output_file,
Feed.link,
General.Get_Title(
Feed.link),
Dump_Types=Dump_Types)
Data_to_Cache.append(Feed.link)
Current_Step += 1
else:
logging.info(General.Date() + " - " +
__name__.strip('plugins.') +
" - Query not found.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
try:
Flickr_Details = Load_Configuration()
flickr_api.set_keys(api_key=Flickr_Details[0],
api_secret=Flickr_Details[1])
except:
logging.info(General.Date() + " - " + __name__.strip('plugins.') +
" - Failed to establish API identity.")
for Query in Query_List:
Email_Regex = re.search(r"[^@]+@[^\.]+\..+", Query)
if Email_Regex:
try:
User = flickr_api.Person.findByEmail(Query)
Photos = User.getPhotos()
General.Main_File_Create(Directory, Plugin_Name, Photos, Query,
".txt")
Output_Connections = General.Connections(
Query, Plugin_Name, "flickr.com", "Data Leakage", Task_ID,
Plugin_Name.lower())
Current_Step = 0
for Photo in Photos:
Photo_URL = "https://www.flickr.com/photos/" + Query + "/" + Photo[
"id"]
if Photo_URL not in Cached_Data and Photo_URL not in Data_to_Cache and Current_Step < int(
Limit):
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Photo_Response = requests.get(Photo_URL,
headers=headers).text
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Photo_Response,
Photo, The_File_Extension)
if Output_file:
Output_Connections.Output(
Output_file, Photo_URL,
General.Get_Title(Photo_URL))
Data_to_Cache.append(Photo_URL)
Current_Step += 1
except:
logging.info(General.Date() + " - " +
__name__.strip('plugins.') +
" - Failed to make API call.")
else:
try:
User = flickr_api.Person.findByUserName(Query)
Photos = User.getPhotos()
General.Main_File_Create(Directory, Plugin_Name, Photos, Query,
".txt")
Output_Connections = General.Connections(
Query, Plugin_Name, "flickr.com", "Data Leakage", Task_ID,
Plugin_Name.lower())
Current_Step = 0
for Photo in Photos:
Photo_URL = "https://www.flickr.com/photos/" + Query + "/" + Photo[
"id"]
if Photo_URL not in Cached_Data and Photo_URL not in Data_to_Cache and Current_Step < int(
Limit):
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Photo_Response = requests.get(Photo_URL,
headers=headers).text
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Photo_Response,
str(Photo['id']), The_File_Extension)
if Output_file:
Output_Connections.Output(
Output_file, Photo_URL,
General.Get_Title(Photo_URL))
Data_to_Cache.append(Photo_URL)
Current_Step += 1
except:
logging.info(General.Date() + " - " +
__name__.strip('plugins.') +
" - Failed to make API call.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID):
Data_to_Cache = []
Cached_Data = []
Configuration_Details = Load_Configuration()
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if Configuration_Details[1].lower() == "true":
Request = 'https://api.certspotter.com/v1/issuances?domain=' + Query + '&include_subdomains=true&expand=dns_names&expand=issuer&expand=cert'
Response = requests.get(Request,
auth=(Configuration_Details[0], '')).text
else:
Request = 'https://api.certspotter.com/v1/issuances?domain=' + Query + '&expand=dns_names&expand=issuer&expand=cert'
Response = requests.get(Request,
auth=(Configuration_Details[0], '')).text
JSON_Response = json.loads(Response)
if 'exists' not in JSON_Response:
if JSON_Response:
if Request not in Cached_Data and Request not in Data_to_Cache:
try:
SSLMate_Regex = re.search(
"([\w\d]+)\.[\w]{2,3}(\.[\w]{2,3})?(\.[\w]{2,3})?",
Query)
if SSLMate_Regex:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name,
json.dumps(JSON_Response,
indent=4,
sort_keys=True),
SSLMate_Regex.group(1), The_File_Extension)
if Output_file:
Output_Connections = General.Connections(
Query, Plugin_Name,
"sslmate.com", "Domain Spoof", Task_ID,
Plugin_Name.lower())
Output_Connections.Output(
Output_file, Request,
General.Get_Title(Request))
else:
logging.warning(
General.Date() + " - " +
__name__.strip('plugins.') +
" - Failed to match regular expression.")
except:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') +
" - Failed to create file.")
Data_to_Cache.append(Request)
else:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') + " - No response.")
else:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') +
" - Query does not exist.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, **kwargs):
Data_to_Cache = []
Cached_Data = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
if Type == "ABN":
Main_URL = 'https://abr.business.gov.au/ABN/View?id=' + Query
Response = requests.get(Main_URL).text
try:
if 'Error searching ABN Lookup' not in Response:
Query = str(int(Query))
if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Response,
General.Get_Title(Main_URL),
The_File_Extension)
if Output_file:
General.Connections(
Output_file, Query, Plugin_Name, Main_URL,
"abr.business.gov.au",
"Data Leakage", Task_ID,
General.Get_Title(Main_URL), Plugin_Name)
Data_to_Cache.append(Main_URL)
except:
logging.warning(General.Date() +
" Invalid query provided for ABN Search.")
elif Type == "ACN":
Main_URL = 'https://abr.business.gov.au/Search/Run'
Data = {
'SearchParameters.SearchText':
Query,
'SearchParameters.AllNames':
'true',
'ctl00%24ContentPagePlaceholder%24SearchBox%24MainSearchButton':
'Search'
}
Response = requests.post(Main_URL, data=Data).text
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
try:
ACN_Regex = re.search(r".*[a-zA-Z].*", Query)
if ACN_Regex:
General.Main_File_Create(Directory, Plugin_Name,
Response, Query,
The_File_Extension)
Current_Step = 0
ABNs_Regex = re.findall(
r"\<input\sid\=\"Results\_NameItems\_\d+\_\_Compressed\"\sname\=\"Results\.NameItems\[\d+\]\.Compressed\"\stype\=\"hidden\"\svalue\=\"(\d{11})\,\d{2}\s\d{3}\s\d{3}\s\d{3}\,0000000001\,Active\,active\,([\d\w\s\&\-\_\.]+)\,Current\,",
Response)
if ABNs_Regex:
for ABN_URL, ACN in ABNs_Regex:
Full_ABN_URL = 'https://abr.business.gov.au/ABN/View?abn=' + ABN_URL
if Full_ABN_URL not in Cached_Data and Full_ABN_URL not in Data_to_Cache and Current_Step < int(
Limit):
ACN = ACN.rstrip()
Current_Response = requests.get(
Full_ABN_URL).text
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name,
str(Current_Response),
ACN.replace(' ', '-'),
The_File_Extension)
if Output_file:
General.Connections(
Output_file, Query, Plugin_Name,
Full_ABN_URL,
"abr.business.gov.au",
"Data Leakage", Task_ID,
General.Get_Title(Full_ABN_URL),
Plugin_Name)
Data_to_Cache.append(Full_ABN_URL)
Current_Step += 1
else:
logging.warning(
General.Date() +
" Response did not match regular expression.")
else:
logging.warning(
General.Date() +
" Query did not match regular expression.")
except:
logging.warning(General.Date() +
" Invalid query provided for ACN Search.")
else:
logging.warning(General.Date() + " Invalid request type.")
except:
logging.warning(General.Date() + " Failed to make request.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, **kwargs):
Data_to_Cache = []
Cached_Data = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
if Type == "NZBN":
Main_URL = 'https://app.companiesoffice.govt.nz/companies/app/ui/pages/companies/search?q=' + Query + '&entityTypes=ALL&entityStatusGroups=ALL&incorpFrom=&incorpTo=&addressTypes=ALL&addressKeyword=&start=0&limit=1&sf=&sd=&advancedPanel=true&mode=advanced#results'
Response = requests.get(Main_URL).text
try:
if 'An error has occurred and the requested action cannot be performed.' not in Response:
Query = str(int(Query))
if Main_URL not in Cached_Data and Main_URL not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, Response, General.Get_Title(Main_URL), The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name, Main_URL, "app.companiesoffice.govt.nz", "Data Leakage", Task_ID, General.Get_Title(Main_URL), Plugin_Name)
Data_to_Cache.append(Main_URL)
except:
logging.warning(General.Date() + " Invalid query provided for NZBN Search.")
elif Type == "NZCN":
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
try:
Main_URL = 'https://app.companiesoffice.govt.nz/companies/app/ui/pages/companies/search?q=' + urllib.parse.quote(
Query) + '&entityTypes=ALL&entityStatusGroups=ALL&incorpFrom=&incorpTo=&addressTypes=ALL&addressKeyword=&start=0&limit=' + str(
Limit) + '&sf=&sd=&advancedPanel=true&mode=advanced#results'
Response = requests.get(Main_URL).text
NZCN_Regex = re.search(r".*[a-zA-Z].*", Query)
if NZCN_Regex:
General.Main_File_Create(Directory, Plugin_Name, Response, Query, The_File_Extension)
NZBNs_Regex = re.findall(r"\<span\sclass\=\"entityName\"\>([\w\d\s\-\_\&\|\!\@\#\$\%\^\*\(\)\.\,]+)\<\/span\>\s<span\sclass\=\"entityInfo\"\>\((\d{6})\)\s\(NZBN\:\s(\d{13})\)", Response)
if NZBNs_Regex:
for NZCN, NZ_ID, NZBN_URL in NZBNs_Regex:
print(NZBN_URL, NZ_ID, NZCN)
Full_NZBN_URL = 'https://app.companiesoffice.govt.nz/companies/app/ui/pages/companies/' + NZ_ID + '?backurl=H4sIAAAAAAAAAEXLuwrCQBCF4bfZNtHESIpBbLQwhWBeYNgddSF7cWai5O2NGLH7zwenyHgjKWwKGaOfSwjZ3ncPaOt1W9bbsmqaamMoqtepnzIJ7Ltu2RdFHeXIacxf9tEmzgdOAZbuExh0jknk%2F17gRNMrsQMjiqxQmsEHr7Aycp3NfY5PjJbcGSMNoDySCckR%2FPwNLgXMiL4AAAA%3D'
if Full_NZBN_URL not in Cached_Data and Full_NZBN_URL not in Data_to_Cache:
Current_Response = requests.get(Full_NZBN_URL).text
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, str(Current_Response), NZCN.replace(' ', '-'), The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name, Full_NZBN_URL, "app.companiesoffice.govt.nz", "Data Leakage", Task_ID, General.Get_Title(Full_NZBN_URL), Plugin_Name)
Data_to_Cache.append(Full_NZBN_URL)
else:
logging.warning(General.Date() + " Response did not match regular expression.")
else:
logging.warning(General.Date() + " Query did not match regular expression.")
except:
logging.warning(General.Date() + " Invalid query provided for NZCN Search.")
else:
logging.warning(General.Date() + " Invalid request type.")
except:
logging.warning(General.Date() + " Failed to make request.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, Type, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if Type == "User":
Local_Plugin_Name = Plugin_Name + "-" + Type
CSE_Response = instagram_explore.user(Query)
CSE_JSON_Output_Response = json.dumps(CSE_Response, indent=4, sort_keys=True)
Output_file = General.Main_File_Create(Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, ".json")
Posts = CSE_Response[0]["edge_owner_to_timeline_media"]["edges"]
Output_Connections = General.Connections(Query, Local_Plugin_Name, "instagram.com", "Data Leakage", Task_ID, Local_Plugin_Name.lower())
Current_Step = 0
for Post in Posts:
Shortcode = Post["node"]["shortcode"]
URL = "https://www.instagram.com/p/" + Shortcode + "/"
if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(Limit):
if Output_file:
Output_Connections.Output(Output_file, URL, General.Get_Title(URL))
Data_to_Cache.append(URL)
Current_Step += 1
elif Type == "Tag":
Local_Plugin_Name = Plugin_Name + "-" + Type
CSE_Response = instagram_explore.tag(Query)
CSE_JSON_Output_Response = json.dumps(CSE_Response, indent=4, sort_keys=True)
Output_file = General.Main_File_Create(Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, ".json")
Posts = CSE_Response[0]["edge_hashtag_to_media"]["edges"]
Output_Connections = General.Connections(Query, Local_Plugin_Name, "instagram.com", "Data Leakage", Task_ID, Local_Plugin_Name.lower())
Current_Step = 0
for Post in Posts:
Shortcode = Post["node"]["shortcode"]
URL = "https://www.instagram.com/p/" + Shortcode + "/"
if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(Limit):
if Output_file:
Output_Connections.Output(Output_file, URL, General.Get_Title(URL))
Data_to_Cache.append(URL)
Current_Step += 1
elif Type == "Location":
Local_Plugin_Name = Plugin_Name + "-" + Type
CSE_Response = location(Query)
CSE_JSON_Output_Response = json.dumps(CSE_Response, indent=4, sort_keys=True)
Output_file = General.Main_File_Create(Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, ".json")
Posts = CSE_Response[0]["edge_location_to_media"]["edges"]
Output_Connections = General.Connections(Query, Local_Plugin_Name, "instagram.com", "Data Leakage", Task_ID, Local_Plugin_Name.lower())
Current_Step = 0
for Post in Posts:
Shortcode = Post["node"]["shortcode"]
URL = "https://www.instagram.com/p/" + Shortcode + "/"
if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(Limit):
if Output_file:
Output_Connections.Output(Output_file, URL, General.Get_Title(URL))
Data_to_Cache.append(URL)
Current_Step += 1
elif Type == "Media":
Local_Plugin_Name = Plugin_Name + "-" + Type
CSE_Response = instagram_explore.media(Query)
if CSE_Response:
CSE_JSON_Output_Response = json.dumps(CSE_Response, indent=4, sort_keys=True)
Output_file = General.Main_File_Create(Directory, Local_Plugin_Name, CSE_JSON_Output_Response, Query, ".json")
URL = "https://www.instagram.com/p/" + Query + "/"
if URL not in Cached_Data and URL not in Data_to_Cache:
if Output_file:
Output_Connections = General.Connections(Query, Local_Plugin_Name, "instagram.com", "Data Leakage", Task_ID, Local_Plugin_Name.lower())
Output_Connections.Output(Output_file, URL, General.Get_Title(URL))
Data_to_Cache.append(URL)
else:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Invalid response.")
else:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Invalid type provided.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Ebay_API_Key = Load_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
API_Request = Connection(appid=Ebay_API_Key, config_file=None)
API_Response = API_Request.execute('findItemsAdvanced',
{'keywords': Query})
JSON_Output_Response = json.dumps(API_Response.dict(),
indent=4,
sort_keys=True)
JSON_Response = json.dumps(API_Response.dict())
JSON_Response = json.loads(JSON_Response)
General.Main_File_Create(Directory, Plugin_Name,
JSON_Output_Response, Query, ".json")
if JSON_Response["ack"] == "Success":
Current_Step = 0
for JSON_Line in JSON_Response['searchResult']['item']:
Ebay_Item_URL = JSON_Line['viewItemURL']
if Ebay_Item_URL not in Cached_Data and Ebay_Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Ebay_Item_Regex = re.search(
r"http\:\/\/www\.ebay\.com\/itm\/([\w\d\-]+)\-\/\d+",
Ebay_Item_URL)
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Ebay_Item_Response = requests.get(Ebay_Item_URL,
headers=headers).text
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Ebay_Item_Response,
Ebay_Item_Regex.group(1), The_File_Extension)
if Output_file:
General.Connections(
Output_file, Query, Plugin_Name, Ebay_Item_URL,
"ebay.com", "Data Leakage", Task_ID,
General.Get_Title(Ebay_Item_URL),
Plugin_Name.lower())
Data_to_Cache.append(Ebay_Item_URL)
Current_Step += 1
else:
logging.warning(General.Date() + " No results found.")
except:
logging.info(General.Date() + " Failed to make API call.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
Tor_Pull_URL = Tor_General_URL + Query
Tor_Scrape_URLs = General.Get_Latest_URLs(Tor_Pull_URL,
Tor_Scrape_Regex_URL)
if Tor_Scrape_URLs:
Output_file = General.Main_File_Create(Directory,
Tor_Plugin_Name.lower(),
"\n".join(Tor_Scrape_URLs),
Query, The_File_Extension)
if Output_file:
Current_Step = 0
for URL in Tor_Scrape_URLs:
if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(
Limit):
General.Connections(Output_file, Query,
Tor_Plugin_Name, URL, "ahmia.fl",
"Domain Spoof", Task_ID,
General.Get_Title(URL),
Plugin_Name.lower())
Data_to_Cache.append(URL)
Current_Step += 1
else:
logging.info(General.Date() + " No Tor links scraped.")
I2P_Pull_URL = I2P_General_URL + Query
I2P_Scrape_URLs = General.Get_Latest_URLs(I2P_Pull_URL,
I2P_Scrape_Regex_URL)
if I2P_Scrape_URLs:
Output_file = General.Main_File_Create(Directory,
I2P_Plugin_Name.lower(),
"\n".join(I2P_Scrape_URLs),
Query, The_File_Extension)
if Output_file:
Current_Step = 0
for URL in I2P_Scrape_URLs:
if URL not in Cached_Data and URL not in Data_to_Cache and Current_Step < int(
Limit):
General.Connections(Output_file, Query,
I2P_Plugin_Name, URL, "ahmia.fl",
"Domain Spoof", Task_ID,
General.Get_Title(URL),
Plugin_Name.lower())
Data_to_Cache.append(URL)
Current_Step += 1
else:
logging.info(General.Date() + " No I2P links scraped.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Transaction_Search(Query_List, Task_ID, Type, **kwargs):
Local_Plugin_Name = Plugin_Name + "-Transaction-Search"
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Local_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
if Type == "btc" or Type == "bch":
Query_Regex = re.search(r"[\d\w]{64}", Query)
elif Type == "eth":
Query_Regex = re.search(r"(0x[\d\w]{64})", Query)
else:
logging.warning(General.Date() + " Invalid type provided.")
if Query_Regex:
Main_URL = "https://www.blockchain.com/" + Type + "/tx/" + Query
Main_Response = requests.get(Main_URL).text
if Type == "btc":
Address_Regex = re.findall(r"\/btc\/address\/([\d\w]{26,34})",
Main_Response)
elif Type == "bch":
Address_Regex = re.findall(r"([\d\w]{42})", Main_Response)
elif Type == "eth":
Address_Regex = re.findall(r"(0x[\w\d]{40})", Main_Response)
else:
logging.warning(General.Date() + " Invalid type provided.")
if Address_Regex:
Current_Step = 0
for Transaction in Address_Regex:
Query_URL = "https://www.blockchain.com/" + Type + "/address/" + Transaction
if Query_URL not in Cached_Data and Query_URL not in Data_to_Cache and Current_Step < int(
Limit):
Transaction_Response = requests.get(Query_URL).text
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Local_Plugin_Name,
Transaction_Response, Transaction,
The_File_Extension)
if Output_file:
General.Connections(Output_file, Query,
Local_Plugin_Name, Query_URL,
"blockchain.com",
"Blockchain Address", Task_ID,
General.Get_Title(Query_URL),
Plugin_Name.lower())
Data_to_Cache.append(Query_URL)
Current_Step += 1
else:
logging.warning(General.Date() +
" Failed to match regular expression.")
else:
logging.warning(General.Date() +
" Failed to match regular expression.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w")
def Search(Query_List, Task_ID, Type_of_Query, **kwargs):
Data_to_Cache = []
Cached_Data = []
try:
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
try:
pyhibp.set_api_key(key=Load_Configuration())
except:
logging.warning(General.Date() + " Failed to set API key, make sure it is set in the configuration file.")
Query_List = General.Convert_to_List(Query_List)
if Type_of_Query == "email":
Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query
Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name)
if not Cached_Data:
Cached_Data = []
for Query in Query_List:
Query_Response = pyhibp.get_pastes(email_address=Query)
logging.info(Query_Response)
if Query_Response:
Domain = Query_Response[0]["Source"]
ID = Query_Response[0]["Id"]
Link = "https://www." + Domain + ".com/" + ID
JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True)
if Link not in Cached_Data and Link not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, JSON_Query_Response, "email", The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower())
Data_to_Cache.append(Link)
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w")
elif Type_of_Query == "breach":
Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query
Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name)
if not Cached_Data:
Cached_Data = []
for Query in Query_List:
Query_Response = pyhibp.get_single_breach(breach_name=Query)
if Query_Response:
Domain = Query_Response["Domain"]
Link = "https://www." + Domain + ".com/"
JSON_Query_Response = json.dumps(Query_Response, indent=4, sort_keys=True)
if Link not in Cached_Data and Link not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "breach", The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower())
Data_to_Cache.append(Link)
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w")
elif Type_of_Query == "password":
Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query
Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name)
if not Cached_Data:
Cached_Data = []
for Query in Query_List:
Query_Response = pw.is_password_breached(password=Query)
logging.info(Query_Response)
if Query_Response:
Link = "https://haveibeenpwned.com/Passwords?" + Query
if Link not in Cached_Data and Link not in Data_to_Cache:
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Plugin_Name, str(Query_Response), "password", ".txt")
if Output_file:
General.Connections(Output_file, Query, Local_Plugin_Name, Link, "haveibeenpwned.com", "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower())
Data_to_Cache.append(Link)
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w")
elif Type_of_Query == "account":
Local_Plugin_Name = Plugin_Name + "-" + Type_of_Query
Cached_Data = General.Get_Cache(Directory, Local_Plugin_Name)
if not Cached_Data:
Cached_Data = []
for Query in Query_List:
Query_Response = pyhibp.get_account_breaches(account=Query, truncate_response=True)
if Query_Response:
Current_Step = 0
for Response in Query_Response:
Current_Response = pyhibp.get_single_breach(breach_name=Response['Name'])
JSON_Query_Response = json.dumps(Current_Response, indent=4, sort_keys=True)
Link = "https://" + Current_Response['Domain']
if Current_Response['Domain'] not in Cached_Data and Current_Response['Domain'] not in Data_to_Cache and Current_Step < int(Limit):
Output_file = General.Create_Query_Results_Output_File(Directory, Query, Local_Plugin_Name, JSON_Query_Response, "account", The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Local_Plugin_Name, Link, Current_Response['Domain'], "Data Leakage", Task_ID, General.Get_Title(Link), Local_Plugin_Name.lower())
Data_to_Cache.append(Current_Response['Domain'])
Current_Step += 1
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Local_Plugin_Name, "w")
else:
logging.warning(General.Date() + " Invalid type provided.")
except:
logging.warning(General.Date() + " Execution error.")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
else:
Limit = 10
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Location = General.Load_Location_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
Response = requests.get("http://itunes.apple.com/search?term=" +
Query + "&country=" + Location +
"&entity=software&limit=" +
str(Limit)).text
except:
logging.warning(
General.Date() + " - " + __name__.strip('plugins.') +
" - Failed to make request, are you connected to the internet?"
)
JSON_Response = json.loads(Response)
General.Main_File_Create(
Directory, "iTunes", json.dumps(Response, indent=4,
sort_keys=True), Query, ".json")
if 'resultCount' in JSON_Response:
if not JSON_Response['resultCount'] == 0:
if JSON_Response['resultCount'] > 0:
Output_Connections = General.Connections(
Query, Plugin_Name, "instagram.com", "Data Leakage",
Task_ID, Concat_Plugin_Name)
for JSON_Object in JSON_Response['results']:
JSON_Object_Response = requests.get(
JSON_Object['artistViewUrl']).text
if JSON_Object[
'artistViewUrl'] not in Cached_Data and JSON_Object[
'artistViewUrl'] not in Data_to_Cache:
iTunes_Regex = re.search(
"https\:\/\/itunes\.apple\.com\/" + Location +
"\/developer\/[\w\d\-]+\/(id[\d]{9,10})\?mt\=\d\&uo\=\d",
JSON_Object['artistViewUrl'])
if iTunes_Regex:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query,
Plugin_Name, JSON_Object_Response,
iTunes_Regex.group(1), The_File_Extension)
if Output_file:
Output_Connections.Output(
Output_file,
JSON_Object['artistViewUrl'],
General.Get_Title(
JSON_Object['artistViewUrl']))
Data_to_Cache.append(JSON_Object['artistViewUrl'])
else:
logging.warning(
General.Date() + " - " + __name__.strip('plugins.') +
" - Invalid value provided, value less than 0.")
else:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') +
" - Invalid value provided, value equal to 0.")
else:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') + " - Invalid value.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
