def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
else:
Limit = 10
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Location = General.Load_Location_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
Response = requests.get("http://itunes.apple.com/search?term=" +
Query + "&country=" + Location +
"&entity=software&limit=" +
str(Limit)).text
except:
logging.warning(
General.Date() + " - " + __name__.strip('plugins.') +
" - Failed to make request, are you connected to the internet?"
)
JSON_Response = json.loads(Response)
General.Main_File_Create(
Directory, "iTunes", json.dumps(Response, indent=4,
sort_keys=True), Query, ".json")
if 'resultCount' in JSON_Response:
if not JSON_Response['resultCount'] == 0:
if JSON_Response['resultCount'] > 0:
Output_Connections = General.Connections(
Query, Plugin_Name, "instagram.com", "Data Leakage",
Task_ID, Concat_Plugin_Name)
for JSON_Object in JSON_Response['results']:
JSON_Object_Response = requests.get(
JSON_Object['artistViewUrl']).text
if JSON_Object[
'artistViewUrl'] not in Cached_Data and JSON_Object[
'artistViewUrl'] not in Data_to_Cache:
iTunes_Regex = re.search(
"https\:\/\/itunes\.apple\.com\/" + Location +
"\/developer\/[\w\d\-]+\/(id[\d]{9,10})\?mt\=\d\&uo\=\d",
JSON_Object['artistViewUrl'])
if iTunes_Regex:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query,
Plugin_Name, JSON_Object_Response,
iTunes_Regex.group(1), The_File_Extension)
if Output_file:
Output_Connections.Output(
Output_file,
JSON_Object['artistViewUrl'],
General.Get_Title(
JSON_Object['artistViewUrl']))
Data_to_Cache.append(JSON_Object['artistViewUrl'])
else:
logging.warning(
General.Date() + " - " + __name__.strip('plugins.') +
" - Invalid value provided, value less than 0.")
else:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') +
" - Invalid value provided, value equal to 0.")
else:
logging.warning(General.Date() + " - " +
__name__.strip('plugins.') + " - Invalid value.")
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
def Search(Query_List, Task_ID, **kwargs):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Location = General.Load_Location_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(kwargs)
for Query in Query_List:
Main_URL = f"https://www.{Domain}/en-{Location}/search?q={Query}"
Win_Store_Response = General.Request_Handler(
Main_URL,
Application_JSON_CT=True,
Accept_XML=True,
Accept_Language_EN_US=True)
Main_File = General.Main_File_Create(Directory, Plugin_Name,
Win_Store_Response, Query,
The_File_Extension)
Win_Store_Regex = re.findall(r"\/en\-au\/p\/([\w\-]+)\/([\w\d]+)",
Win_Store_Response)
Output_Connections = General.Connections(Query, Plugin_Name,
Domain, "Application",
Task_ID,
Concat_Plugin_Name)
if Win_Store_Regex:
Current_Step = 0
for Regex_Group_1, Regex_Group_2 in Win_Store_Regex:
Item_URL = f"https://www.microsoft.com/en-au/p/{Regex_Group_1}/{Regex_Group_2}"
Win_Store_Responses = General.Request_Handler(
Item_URL,
Application_JSON_CT=True,
Accept_XML=True,
Accept_Language_EN_US=True,
Filter=True,
Host=f"https://www.{Domain}")
Win_Store_Response = Win_Store_Responses["Filtered"]
Title = "Windows Store | " + General.Get_Title(Item_URL)
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Win_Store_Response,
Regex_Group_1, The_File_Extension)
if Output_file:
Output_Connections.Output([Main_File, Output_file],
Item_URL, Title,
Concat_Plugin_Name)
Data_to_Cache.append(Item_URL)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
Current_Step += 1
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression."
)
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Location = General.Load_Location_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(kwargs)
for Query in Query_List:
try:
Request_Query = urllib.parse.quote(Query)
Main_URL = f"http://{Domain}/search?term={Request_Query}&country={Location}&entity=software&limit={str(Limit)}"
Response = General.Request_Handler(Main_URL)
except:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to make request, are you connected to the internet?"
)
break
JSON_Response = json.loads(Response)
Main_File = General.Main_File_Create(
Directory, "iTunes",
json.dumps(JSON_Response, indent=4, sort_keys=True), Query,
The_File_Extensions["Main"])
if 'resultCount' in JSON_Response:
if JSON_Response['resultCount'] > 0:
Output_Connections = General.Connections(
Query, Plugin_Name, Domain, "Application", Task_ID,
Concat_Plugin_Name)
for JSON_Object in JSON_Response['results']:
JSON_Object_Responses = General.Request_Handler(
JSON_Object['artistViewUrl'],
Filter=True,
Host=f"https://{Domain}")
JSON_Object_Response = JSON_Object_Responses[
"Filtered"]
if JSON_Object[
'artistViewUrl'] not in Cached_Data and JSON_Object[
'artistViewUrl'] not in Data_to_Cache:
iTunes_Regex = re.search(
r"https\:\/\/apps\.apple\.com\/" +
rf"{Location}" +
r"\/developer\/[\w\d\-]+\/(id[\d]{9,10})\?.+",
JSON_Object['artistViewUrl'])
if iTunes_Regex:
Output_file = General.Create_Query_Results_Output_File(
Directory, Query,
Plugin_Name, JSON_Object_Response,
iTunes_Regex.group(1),
The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output(
[Main_File, Output_file],
JSON_Object['artistViewUrl'],
General.Get_Title(
JSON_Object['artistViewUrl']),
Concat_Plugin_Name)
Data_to_Cache.append(
JSON_Object['artistViewUrl'])
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Invalid value provided, value not greater than 0."
)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Invalid value."
)
General.Write_Cache(Directory, Cached_Data, Data_to_Cache, Plugin_Name)
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs):
try:
Data_to_Cache = []
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Location = General.Load_Location_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(kwargs)
for Query in Query_List:
Main_URL = f"https://www.microsoft.com/en-{Location}/search?q={Query}"
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Win_Store_Response = requests.get(Main_URL, headers=headers).text
Main_File = General.Main_File_Create(Directory, Plugin_Name,
Win_Store_Response, Query,
The_File_Extension)
Win_Store_Regex = re.findall(r"\/en\-au\/p\/([\w\-]+)\/([\w\d]+)",
Win_Store_Response)
Output_Connections = General.Connections(Query, Plugin_Name,
"microsoft.com",
"Application", Task_ID,
Concat_Plugin_Name)
if Win_Store_Regex:
Current_Step = 0
for Regex_Group_1, Regex_Group_2 in Win_Store_Regex:
Item_URL = f"https://www.microsoft.com/en-au/p/{Regex_Group_1}/{Regex_Group_2}"
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Win_Store_Response = requests.get(Item_URL,
headers=headers).text
Title = "Windows Store | " + General.Get_Title(Item_URL)
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Win_Store_Response,
Regex_Group_1, The_File_Extension)
if Output_file:
Output_Connections.Output([Main_File, Output_file],
Item_URL, Title,
Concat_Plugin_Name)
Data_to_Cache.append(Item_URL)
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to create output file. File may already exist."
)
Current_Step += 1
else:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - Failed to match regular expression."
)
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
except Exception as e:
logging.warning(
f"{General.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def Search(Query_List, Task_ID, **kwargs):
Data_to_Cache = []
Cached_Data = []
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Concat_Plugin_Name)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Concat_Plugin_Name)
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Location = General.Load_Location_Configuration()
Cached_Data = General.Get_Cache(Directory, Plugin_Name)
if not Cached_Data:
Cached_Data = []
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
Main_URL = "https://www.microsoft.com/en-" + Location + "/search?q=" + Query
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Win_Store_Response = requests.get(Main_URL, headers=headers).text
General.Main_File_Create(Directory, Plugin_Name, Win_Store_Response,
Query, The_File_Extension)
Win_Store_Regex = re.findall(r"\/en\-au\/p\/([\w\-]+)\/([\w\d]+)",
Win_Store_Response)
if Win_Store_Regex:
Current_Step = 0
for Regex_Group_1, Regex_Group_2 in Win_Store_Regex:
Item_URL = "https://www.microsoft.com/en-au/p/" + Regex_Group_1 + "/" + Regex_Group_2
headers = {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0',
'Accept':
'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5'
}
Win_Store_Response = requests.get(Item_URL,
headers=headers).text
if Item_URL not in Cached_Data and Item_URL not in Data_to_Cache and Current_Step < int(
Limit):
Output_file = General.Create_Query_Results_Output_File(
Directory, Query, Plugin_Name, Win_Store_Response,
Regex_Group_1, The_File_Extension)
if Output_file:
General.Connections(Output_file, Query, Plugin_Name,
Item_URL, "microsoft.com",
"Data Leakage", Task_ID,
General.Get_Title(Item_URL),
Concat_Plugin_Name)
Data_to_Cache.append(Item_URL)
Current_Step += 1
if Cached_Data:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "a")
else:
General.Write_Cache(Directory, Data_to_Cache, Plugin_Name, "w")
logging.info('Windows Store Search Plugin Terminated.')