def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir,
'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') +
" - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for Flickr_Details in Configuration_Data[Plugin_Name.lower()]:
if Flickr_Details['api_key'] and Flickr_Details['api_secret']:
return [
Flickr_Details['api_key'], Flickr_Details['api_secret']
]
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') +
" - Failed to load location details.")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir,
'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') +
" - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for Reddit_Details in Configuration_Data[Plugin_Name.lower()]:
Reddit_Client_ID = Reddit_Details['client_id']
Reddit_Client_Secret = Reddit_Details['client_secret']
Reddit_User_Agent = Reddit_Details['user_agent']
Reddit_Username = Reddit_Details['username']
Reddit_Password = Reddit_Details['password']
Subreddit_to_Search = Reddit_Details["subreddits"]
if Reddit_Client_ID and Reddit_Client_Secret and Reddit_User_Agent and Reddit_Username and Reddit_Password and Subreddit_to_Search:
return [
Reddit_Client_ID, Reddit_Client_Secret,
Reddit_User_Agent, Reddit_Username, Reddit_Password,
Subreddit_to_Search
]
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') +
" - Failed to load Reddit details.")
def Search(Query_List, Task_ID, **kwargs):
if kwargs.get('Limit'):
if int(kwargs["Limit"]) > 0:
Limit = kwargs["Limit"]
else:
Limit = 10
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Twitter_Credentials = Load_Configuration()
Query_List = General.Convert_to_List(Query_List)
for Query in Query_List:
try:
Authentication = tweepy.OAuthHandler(Twitter_Credentials[0], Twitter_Credentials[1])
Authentication.set_access_token(Twitter_Credentials[2], Twitter_Credentials[3])
API = tweepy.API(Authentication)
General_Pull(Query, Limit, Directory, API, Task_ID)
except:
logging.info(General.Date() + " Failed to get results. Are you connected to the internet?")
def Search(self):
try:
Directory = General.Make_Directory(self.Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, self.Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Twitter_Credentials = self.Load_Configuration()
for Query in self.Query_List:
try:
Authentication = tweepy.OAuthHandler(Twitter_Credentials[0], Twitter_Credentials[1])
Authentication.set_access_token(Twitter_Credentials[2], Twitter_Credentials[3])
API = tweepy.API(Authentication)
self.General_Pull(Query, Directory, API)
except:
logging.info(f"{Common.Date()} - {self.Logging_Plugin_Name} - Failed to get results. Are you connected to the internet?")
except Exception as e:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def Load_Configuration():
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Configuration_File = os.path.join(File_Dir, 'plugins/common/config/config.json')
logging.info(General.Date() + " - " + __name__.strip('plugins.') + " - Loading configuration data.")
try:
with open(Configuration_File) as JSON_File:
Configuration_Data = json.load(JSON_File)
for YouTube_Details in Configuration_Data[Plugin_Name.lower()]:
YouTube_Developer_Key = YouTube_Details['developer_key']
YouTube_Application_Name = YouTube_Details['application_name']
YouTube_Application_Version = YouTube_Details['application_version']
YouTube_Location = YouTube_Details['location']
YouTube_Location_Radius = YouTube_Details['location_radius']
if YouTube_Developer_Key and YouTube_Application_Name and YouTube_Application_Version and YouTube_Location and YouTube_Location_Radius:
return [YouTube_Developer_Key, YouTube_Application_Name, YouTube_Application_Version, YouTube_Location, YouTube_Location_Radius]
else:
return None
except:
logging.warning(General.Date() + " - " + __name__.strip('plugins.') + " - Failed to load location details.")
def Search(Query_List, Task_ID, Limit=10):
try:
Directory = General.Make_Directory(Plugin_Name.lower())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Log_File = General.Logging(Directory, Plugin_Name.lower())
handler = logging.FileHandler(os.path.join(Directory, Log_File), "w")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
Twitter_Credentials = Load_Configuration()
Query_List = General.Convert_to_List(Query_List)
Limit = General.Get_Limit(Limit)
for Query in Query_List:
try:
Authentication = tweepy.OAuthHandler(Twitter_Credentials[0],
Twitter_Credentials[1])
Authentication.set_access_token(Twitter_Credentials[2],
Twitter_Credentials[3])
API = tweepy.API(Authentication)
General_Pull(Query, Limit, Directory, API, Task_ID)
except:
logging.info(
f"{Common.Date()} - {__name__.strip('plugins.')} - Failed to get results. Are you connected to the internet?"
)
except Exception as e:
logging.warning(
f"{Common.Date()} - {__name__.strip('plugins.')} - {str(e)}")
def __init__(self, Query_List, Task_ID, Limit=10):
self.Plugin_Name = "Flickr"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "flickr.com"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "SSLMate"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".json"
self.Domain = "sslmate.com"
self.Result_Type = "Certificate Details"
def __init__(self, Query_List, Task_ID, Type):
self.Plugin_Name = "VirusTotal"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "virustotal.com"
self.Type = Type
def __init__(self, Query_List, Task_ID, Limit=10):
self.The_File_Extension = ".html"
self.Plugin_Name = "RSS"
self.Logging_Plugin_Name = self.Plugin_Name + " Feed Search"
self.Result_Type = "News Report"
self.Limit = General.Get_Limit(Limit)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "Kik"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".html"
self.Domain = "kik.me"
self.Result_Type = "Social Media - Person"
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "Email Verification"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Main_Converted": ".html"}
self.Concat_Plugin_Name = "emailverify"
self.Domain = "verify-email.org"
self.Result_Type = "Email Information"
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "BuiltWith"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "builtwith.com"
self.Result_Type = "Web Application Architecture"
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "IPStack"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Main_Converted": ".html"}
self.Domain = "ipstack.com"
self.Result_Type = "IP Address Information"
def __init__(self, Query_List, Task_ID, Limit=10):
self.Plugin_Name = "DuckDuckGo"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "duckduckgo.com"
self.Result_Type = "Search Result"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "DNS Reconnaissance"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Concat_Plugin_Name = "dnsrecon"
self.Result_Type = "Domain Information"
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "Tumblr"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "tumblr.com"
self.Result_Type = "Social Media - Page"
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "CRT"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".html"
self.Domain = "crt.sh"
self.Result_Type = "Certificate Details"
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "BSB"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".html"
self.Domain = "bsbnumbers.com"
self.Result_Type = "BSB Details"
def __init__(self, Query_List, Task_ID, Type):
self.Plugin_Name = "Threat Crowd"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "threatcrowd.org"
self.Type = Type
def General_Pull(self, Handle, Directory, API):
try:
Data_to_Cache = []
JSON_Response = []
Cached_Data_Object = General.Cache(Directory, self.Plugin_Name)
Cached_Data = Cached_Data_Object.Get_Cache()
Latest_Tweets = API.user_timeline(screen_name=Handle, count=self.Limit)
for Tweet in Latest_Tweets:
try:
JSON_Response.append({
'id': Tweet.id,
'text': Tweet.text,
'author_name': Tweet.user.screen_name,
'url': Tweet.entities['urls'][0]["expanded_url"]
})
except:
JSON_Response.append({
'id': Tweet.id,
'text': Tweet.text,
'author_name': Tweet.user.screen_name
})
JSON_Output = Common.JSON_Handler(JSON_Response).Dump_JSON()
Output_Connections = General.Connections(Handle, self.Plugin_Name, self.Domain, self.Result_Type, self.Task_ID, self.Plugin_Name.lower())
Main_File = General.Main_File_Create(Directory, self.Plugin_Name, JSON_Output, Handle, self.The_File_Extensions["Main"])
for JSON_Item in JSON_Response:
if all(Item in JSON_Item for Item in ['id', 'url', 'text']):
Link = JSON_Item['url']
if Link not in Cached_Data and Link not in Data_to_Cache:
Title = "Twitter | " + JSON_Item['text']
Item_Responses = Common.Request_Handler(Link, Filter=True, Host=f"https://{self.Domain}")
Item_Response = Item_Responses["Filtered"]
Output_file = General.Create_Query_Results_Output_File(Directory, Handle, self.Plugin_Name, Item_Response, str(JSON_Item['id']), self.The_File_Extensions["Query"])
if Output_file:
Output_Connections.Output([Main_File, Output_file], Link, Title, self.Plugin_Name.lower())
Data_to_Cache.append(Link)
else:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Output file not returned.")
else:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - Insufficient parameters provided.")
Cached_Data_Object.Write_Cache(Data_to_Cache)
except Exception as e:
logging.warning(f"{Common.Date()} - {self.Logging_Plugin_Name} - {str(e)}")
def __init__(self, Query_List, Task_ID, Limit=10):
self.Plugin_Name = "Torrent"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".json"
self.Domain = "thepiratebay.org"
self.Result_Type = "Torrent"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID, Limit=10):
self.Plugin_Name = "Windows Store"
self.Concat_Plugin_Name = "windowsstore"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".html"
self.Domain = "microsoft.com"
self.Result_Type = "Application"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "Greynoise IP"
self.Concat_Plugin_Name = "greynoise"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "greynoise.io"
self.Result_Type = "IP Address Information"
def __init__(self, Query_List, Task_ID, Limit=10):
self.Plugin_Name = "Library Genesis"
self.Concat_Plugin_Name = "libgen"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".html"
self.Domain = "gen.lib.rus.ec"
self.Result_Type = "Publication"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID, Limit=10):
self.Plugin_Name = "Reddit"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extension = ".html"
self.Domain = "reddit.com"
self.Result_Type = "Forum"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "Doing Business"
self.Concat_Plugin_Name = "doingbusiness"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "doingbusiness.org"
self.Result_Type = "Economic Details"
def __init__(self, Query_List, Task_ID, Limit=10):
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Plugin_Name = "YouTube"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.Domain = "youtube.com"
self.Result_Type = "Social Media - Media"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID, Limit=10):
self.Plugin_Name = "Phishstats"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Domain = "phishstats.info"
self.Result_Type = "Phishing"
self.Limit = General.Get_Limit(Limit)
def __init__(self, Query_List, Task_ID):
self.Plugin_Name = "Email Reputation"
self.Logging_Plugin_Name = General.Get_Plugin_Logging_Name(
self.Plugin_Name)
self.Task_ID = Task_ID
self.Query_List = General.Convert_to_List(Query_List)
self.The_File_Extensions = {"Main": ".json", "Query": ".html"}
self.Concat_Plugin_Name = "emailrep"
self.Domain = "emailrep.io"
self.Result_Type = "Email Information"