def Load_Configuration(self):
logging.info(f"{Common.Date()} - {self.Logging_Plugin_Name} - Loading configuration data.")
Result = Common.Configuration(Input=True).Load_Configuration(Location=True, Object="general", Details_to_Load=["location"])
if Result:
return Result
else:
return None
def Load_Configuration(self):
logging.info(f"{Common.Date()} - {self.Logging_Plugin_Name} - Loading configuration data.")
Result = Common.Configuration(Input=True).Load_Configuration(Object=self.Plugin_Name.lower(), Details_to_Load=["api_key"])
if Result:
return Result
else:
return None
def Load_Configuration(self):
logging.info(f"{Common.Date()} - {self.Logging_Plugin_Name} - Loading configuration data.")
Result = Common.Configuration(Input=True).Load_Configuration(Object=self.Concat_Plugin_Name, Details_to_Load=["api_key"])
if Result:
return base64.b64encode(Result.encode('ascii'))
else:
return None
def Load_Configuration():
logging.info(f"{Common.Date()} - {__name__.strip('plugins.')} - Loading configuration data.")
Connector_Object = Common.Configuration(Input=True)
Result = Connector_Object.Load_Configuration(Object=Plugin_Name.lower(), Details_to_Load=["api_key"])
if Result:
return Result
else:
return None
def Load_Configuration():
logging.info(f"{Common.Date()} - {__name__.strip('plugins.')} - Loading configuration data.")
Connector_Object = Common.Configuration(Input=True)
Result = Connector_Object.Load_Configuration(Object=Concat_Plugin_Name, Details_to_Load=["api_key"])
if Result:
return base64.b64encode(Result.encode('ascii'))
else:
return None
def __init__(self, File_Path, Internally_Requested=False, **kwargs):
self.Internally_Requested = Internally_Requested
self.Chrome_Config = Common.Configuration(
Core=True).Load_Configuration(
Object="google_chrome",
Details_to_Load=["application_path", "chromedriver_path"])
self.File_Path = File_Path
self.Connection = Common.Configuration(Output=True).Load_Configuration(
Postgres_Database=True, Object="postgresql")
self.Cursor = self.Connection.cursor()
if not self.Internally_Requested and kwargs.get(
'Screenshot_ID') and kwargs.get('Screenshot_User'):
self.Screenshot_ID = kwargs['Screenshot_ID']
self.Screenshot_User = kwargs['Screenshot_User']
elif self.Internally_Requested and kwargs.get('Screenshot_Link'):
self.Screenshot_ID = False
self.Screenshot_User = False
self.Screenshot_Link = kwargs['Screenshot_Link']
def Load_Configuration(self):
logging.info(
f"{Common.Date()} - {self.Logging_Plugin_Name} - Loading configuration data."
)
Result = Common.Configuration(Input=True).Load_Configuration(
Object=self.Plugin_Name.lower(),
Details_to_Load=[
"client_id", "client_secret", "user_agent", "username",
"password", "subreddits"
])
if Result:
return Result
else:
return None
def Load_Configuration():
logging.info(
f"{Common.Date()} - {__name__.strip('plugins.')} - Loading configuration data."
)
Connector_Object = Common.Configuration(Input=True)
Result = Connector_Object.Load_Configuration(Object=Plugin_Name.lower(),
Details_to_Load=[
"client_id",
"client_secret",
"user_agent", "username",
"password", "subreddits"
])
if Result:
return Result
else:
return None
def Call_Plugin(self):
Object = Common.Configuration(Output=True)
self.Starter(Object)
try:
Plugin = plugin_verifier.Plugin_Verifier(
self.plugin_name, self.task_id, self.query,
self.limit).Verify_Plugin()
if Plugin and all(Item in Plugin for Item in
["Object", "Search Option", "Function Kwargs"]):
getattr(Plugin["Object"],
Plugin["Search Option"])(**Plugin["Function Kwargs"])
else:
print(
f"{Common.Date()} - Plugin Caller - Failed to start plugin."
)
except Exception as e:
print(f"{Common.Date()} - Plugin Caller - {str(e)}")
finally:
self.Stopper(Object)
def Call_Plugin(self):
Object = Common.Configuration(Output=True)
Starter(Object, self.task_id)
Plugins_Dictionary = {
"YouTube Search": {
"Module": "plugins.YouTube_Search",
"Limit": True
},
"Yandex Search": {
"Module": "plugins.Yandex_Search",
"Limit": True
},
"Windows Store Search": {
"Module": "plugins.Windows_Store_Search",
"Limit": True
},
"Vulners Search": {
"Module": "plugins.Vulners_Search",
"Limit": True
},
"Virus Total Search - Domain": {
"Module": "plugins.Virus_Total_Search",
"Type": "Domain"
},
"Virus Total Search - URL": {
"Module": "plugins.Virus_Total_Search",
"Type": "URL"
},
"Virus Total Search - IP Address": {
"Module": "plugins.Virus_Total_Search",
"Type": "IP"
},
"Virus Total Search - File Hash": {
"Module": "plugins.Virus_Total_Search",
"Type": "Hash"
},
"Vkontakte - User Search": {
"Module": "plugins.Vkontakte_Search",
"Type": "User",
"Limit": True
},
"Vkontakte - Group Search": {
"Module": "plugins.Vkontakte_Search",
"Type": "Group",
"Limit": True
},
"Vehicle Registration Search": {
"Module": "plugins.Vehicle_Registration_Search"
},
"Username Search": {
"Module": "plugins.Username_Search"
},
"Twitter Search": {
"Module": "plugins.Twitter_Search",
"Limit": True
},
"Tumblr Search": {
"Module": "plugins.Tumblr_Search"
},
"Torrent Search": {
"Module": "plugins.Torrent_Search",
"Limit": True
},
"Threat Crowd - Virus Report Search": {
"Module": "plugins.Threat_Crowd_Search",
"Type": "Virus Report"
},
"Threat Crowd - IP Address Search": {
"Module": "plugins.Threat_Crowd_Search",
"Type": "IP Address"
},
"Threat Crowd - Email Search": {
"Module": "plugins.Threat_Crowd_Search",
"Type": "Email"
},
"Threat Crowd - Domain Search": {
"Module": "plugins.Threat_Crowd_Search",
"Type": "Domain"
},
"Threat Crowd - Antivirus Search": {
"Module": "plugins.Threat_Crowd_Search",
"Type": "AV"
},
"Shodan Search - Query": {
"Module": "plugins.Shodan_Search",
"Type": "Search",
"Limit": True
},
"Shodan Search - IP Address": {
"Module": "plugins.Shodan_Search",
"Type": "Host"
},
"RSS Feed Search": {
"Module": "plugins.RSS_Feed_Search",
"Limit": True
},
"Reddit Search": {
"Module": "plugins.Reddit_Search",
"Limit": True
},
"Phone Search - SIM Number": {
"Module": "plugins.Phone_Search",
"Type": "SIM"
},
"Phone Search - ISPC Number": {
"Module": "plugins.Phone_Search",
"Type": "ISPC"
},
"Phone Search - IMSI Number": {
"Module": "plugins.Phone_Search",
"Type": "IMSI"
},
"Phone Search - IMEI Number": {
"Module": "plugins.Phone_Search",
"Type": "IMEI"
},
"Phone Search - Cellular Number": {
"Module": "plugins.Phone_Search",
"Type": "Number"
},
"Phishstats Search": {
"Module": "plugins.Phishstats_Search",
"Limit": True
},
"Pinterest - Pin Search": {
"Module": "plugins.Pinterest_Search",
"Type": "pin",
"Limit": True
},
"Pinterest - Board Search": {
"Module": "plugins.Pinterest_Search",
"Type": "board",
"Limit": True
},
"OK Search - User": {
"Module": "plugins.OK_Search",
"Type": "User"
},
"OK Search - Group": {
"Module": "plugins.OK_Search",
"Type": "Group"
},
"Naver Search": {
"Module": "plugins.Naver_Search",
"Limit": True
},
"Library Genesis Search": {
"Module": "plugins.Library_Genesis_Search",
"Limit": True
},
"Kik Search": {
"Module": "plugins.Kik_Search"
},
"IP Stack Search": {
"Module": "plugins.IPStack_Search"
},
"IntelligenceX Search": {
"Module": "plugins.IntelligenceX_Search",
"Limit": True
},
"Instagram - User Search": {
"Module": "plugins.Instagram_Search",
"Type": "User",
"Limit": True
},
"Instagram - Tag Search": {
"Module": "plugins.Instagram_Search",
"Type": "Tag",
"Limit": True
},
"Instagram - Post Search": {
"Module": "plugins.Instagram_Search",
"Type": "Post"
},
"Hunter Search - Email": {
"Module": "plugins.Hunter_Search",
"Type": "Email",
"Limit": True
},
"Hunter Search - Domain": {
"Module": "plugins.Hunter_Search",
"Type": "Domain",
"Limit": True
},
"Have I Been Pwned - Password Search": {
"Module": "plugins.Have_I_Been_Pwned",
"Type": "password"
},
"Have I Been Pwned - Email Search": {
"Module": "plugins.Have_I_Been_Pwned",
"Type": "email"
},
"Have I Been Pwned - Breach Search": {
"Module": "plugins.Have_I_Been_Pwned",
"Type": "breach"
},
"Have I Been Pwned - Account Search": {
"Module": "plugins.Have_I_Been_Pwned",
"Type": "account"
},
"Greynoise IP Search": {
"Module": "plugins.Greynoise_IP_Search"
},
"Google Search": {
"Module": "plugins.Google_Search",
"Limit": True
},
"Google Play Store Search": {
"Module": "plugins.Google_Play_Store_Search",
"Limit": True
},
"Flickr Search": {
"Module": "plugins.Flickr_Search",
"Limit": True
},
"Email Verification Search": {
"Module": "plugins.Email_Verification_Search"
},
"Email Reputation Search": {
"Module": "plugins.Email_Reputation_Search"
},
"Ebay Search": {
"Module": "plugins.Ebay_Search",
"Limit": True
},
"Domain Fuzzer - Regular Domain Suffixes": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Regular_Extensions",
"Is_Object": True
},
"Domain Fuzzer - Global Domain Suffixes": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Global_Extensions",
"Is_Object": True
},
"Domain Fuzzer - Punycode (Latin Comprehensive)": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Character_Switch",
"Is_Object": True,
"Alphabets": "Latin",
"Comprehensive": True
},
"Domain Fuzzer - Punycode (Latin Condensed)": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Character_Switch",
"Is_Object": True,
"Alphabets": "Latin"
},
"Domain Fuzzer - Punycode (Asian)": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Character_Switch",
"Is_Object": True,
"Alphabets": "Asian"
},
"Domain Fuzzer - Punycode (Middle Eastern)": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Character_Switch",
"Is_Object": True,
"Alphabets": "Middle Eastern"
},
"Domain Fuzzer - Punycode (North African)": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Character_Switch",
"Is_Object": True,
"Alphabets": "North African"
},
"Domain Fuzzer - Punycode (Native American)": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "Character_Switch",
"Is_Object": True,
"Alphabets": "Native American"
},
"Domain Fuzzer - All Extensions": {
"Module": "plugins.Domain_Fuzzer",
"Custom_Search": "All_Extensions",
"Is_Object": True
},
"Doing Business Search": {
"Module": "plugins.Doing_Business_Search"
},
"DNS Reconnaissance Search": {
"Module": "plugins.DNS_Recon_Search"
},
"Default Password Search": {
"Module": "plugins.Default_Password_Search",
"Limit": True
},
"DuckDuckGo Search": {
"Module": "plugins.DuckDuckGo_Search",
"Limit": True
},
"Craigslist Search": {
"Module": "plugins.Craigslist_Search",
"Limit": True
},
"Certificate Transparency - SSLMate": {
"Module": "plugins.Certificate_Transparency"
},
"Certificate Transparency - CRT.sh": {
"Module": "plugins.Certificate_Transparency_CRT"
},
"Builtwith Search": {
"Module": "plugins.BuiltWith_Search"
},
"Business Search - United Kingdom Business Number": {
"Module": "plugins.UK_Business_Search",
"Type": "UKBN"
},
"Business Search - United Kingdom Company Name": {
"Module": "plugins.UK_Business_Search",
"Type": "UKCN",
"Limit": True
},
"Business Search - New Zealand Business Number": {
"Module": "plugins.NZ_Business_Search",
"Type": "NZBN"
},
"Business Search - New Zealand Company Name": {
"Module": "plugins.NZ_Business_Search",
"Type": "NZCN",
"Limit": True
},
"Business Search - Canadian Business Number": {
"Module": "plugins.Canadian_Business_Search",
"Type": "CBN"
},
"Business Search - Canadian Company Name": {
"Module": "plugins.Canadian_Business_Search",
"Type": "CCN",
"Limit": True
},
"Business Search - Australian Business Number": {
"Module": "plugins.Australian_Business_Search",
"Type": "ABN"
},
"Business Search - Australian Company Name": {
"Module": "plugins.Australian_Business_Search",
"Type": "ACN",
"Limit": True
},
"Business Search - American Central Index Key": {
"Module": "plugins.American_Business_Search",
"Type": "CIK"
},
"Business Search - American Company Name": {
"Module": "plugins.American_Business_Search",
"Type": "ACN",
"Limit": True
},
"BSB Search": {
"Module": "plugins.BSB_Search"
},
"Blockchain - Monero Transaction Search": {
"Module": "plugins.Blockchain_Search",
"Custom_Search": "Transaction_Search",
"Type": "monero"
},
"Blockchain - Ethereum Transaction Search": {
"Module": "plugins.Blockchain_Search",
"Custom_Search": "Transaction_Search",
"Type": "eth"
},
"Blockchain - Bitcoin Cash Transaction Search": {
"Module": "plugins.Blockchain_Search",
"Custom_Search": "Transaction_Search",
"Type": "bch"
},
"Blockchain - Bitcoin Transaction Search": {
"Module": "plugins.Blockchain_Search",
"Custom_Search": "Transaction_Search",
"Type": "btc"
},
"Blockchain - Ethereum Address Search": {
"Module": "plugins.Blockchain_Search",
"Custom_Search": "Address_Search",
"Type": "eth"
},
"Blockchain - Bitcoin Cash Address Search": {
"Module": "plugins.Blockchain_Search",
"Custom_Search": "Address_Search",
"Type": "bch"
},
"Blockchain - Bitcoin Address Search": {
"Module": "plugins.Blockchain_Search",
"Custom_Search": "Address_Search",
"Type": "btc"
},
"Apple Store Search": {
"Module": "plugins.Apple_Store_Search",
"Limit": True
},
"Ahmia Darkweb Search": {
"Module": "plugins.Ahmia_Darkweb_Search",
"Limit": True
}
}
try:
if self.plugin_name in Plugins_Dictionary:
Dict_Item = Plugins_Dictionary[self.plugin_name]
Kwargs = {}
for Key in ["Limit", "Type", "Alphabets", "Comprehensive"]:
if Key in Dict_Item and Key != "Limit":
Kwargs[Key] = Dict_Item[Key]
elif Key in Dict_Item and Key == "Limit":
Kwargs[Key] = self.limit
if "Is_Object" not in Dict_Item:
def Inner_Function_Call(self, To_Import, Search_Option,
**kwargs):
Plugin = importlib.import_module(To_Import)
getattr(Plugin, Search_Option)(self.query,
self.task_id, **kwargs)
if "Custom_Search" in Dict_Item:
Search_Option = Dict_Item["Custom_Search"]
else:
Search_Option = "Search"
Inner_Function_Call(self, Dict_Item["Module"],
Search_Option, **Kwargs)
else:
def Inner_Function_Call(Domain_Fuzz_Obj, **kwargs):
getattr(Domain_Fuzz_Obj,
Dict_Item["Custom_Search"])(**kwargs)
Class = importlib.import_module(Dict_Item["Module"])
Domain_Fuzz_Obj = Class.Fuzzer(self.query, self.task_id)
Inner_Function_Call(Domain_Fuzz_Obj, Search_Option,
**Kwargs)
else:
print(f"{Common.Date()} - Invalid plugin provided - {e}")
except Exception as e:
print(f"{Common.Date()} - Plugin Caller Error - {e}")
finally:
Stopper(Object, self.task_id)
'--task',
help=
'This option is used to specify a task ID to run. ./plugin_caller.py -t 1'
)
Arguments = Parser.parse_args()
Scrummage_Working_Directory = pathlib.Path(__file__).parent.absolute()
os.chdir(Scrummage_Working_Directory)
Task_ID = 0
if str(Scrummage_Working_Directory) == str(os.getcwd()):
if Arguments.task:
try:
Task_ID = int(Arguments.task)
Connector_Object = Common.Configuration(Output=True)
Connection = Connector_Object.Load_Configuration(
Postgres_Database=True, Object="postgresql")
cursor = Connection.cursor()
PSQL_Select_Query = 'SELECT * FROM tasks WHERE task_id = %s;'
cursor.execute(PSQL_Select_Query, (Task_ID, ))
result = cursor.fetchone()
if result and Output_API_Checker(result[2]):
Plugin_to_Call = Plugin_Caller(Plugin_Name=result[2],
Limit=result[5],
Task_ID=Task_ID,
Query=result[1])
Plugin_to_Call.Call_Plugin()
except:
help=
'This option is used to specify a task ID to run. ./plugin_caller.py -t 1'
)
Arguments = Parser.parse_args()
Scrummage_Working_Directory = pathlib.Path(__file__).parent.absolute()
os.chdir(Scrummage_Working_Directory)
Task_ID = 0
if str(Scrummage_Working_Directory) == str(os.getcwd()):
if Arguments.task:
try:
Task_ID = int(Arguments.task)
Connection = Common.Configuration(
Output=True).Load_Configuration(Postgres_Database=True,
Object="postgresql")
cursor = Connection.cursor()
PSQL_Select_Query = 'SELECT * FROM tasks WHERE task_id = %s;'
cursor.execute(PSQL_Select_Query, (Task_ID, ))
result = cursor.fetchone()
if result:
Plugin_Caller(Result=result, Task_ID=Task_ID).Call_Plugin()
except:
sys.exit("[-] Invalid Task ID.")
else:
sys.exit("[-] No task provided.")
def Output(self,
Complete_File_List,
Link,
DB_Title,
Directory_Plugin_Name,
Dump_Types=[]):
try:
try:
Text_Complete_Files = "\n- ".join(Complete_File_List)
if type(Dump_Types) == list and len(Dump_Types) > 0:
self.Dump_Types = Dump_Types
Joined_Dump_Types = ", ".join(self.Dump_Types)
self.Title = f"Data for input: {self.Input}, found by Scrummage plugin {self.Plugin_Name}.\nData types include: {Joined_Dump_Types}.\nAll data is stored in\n- {Text_Complete_Files}."
self.Ticket_Subject = f"Scrummage {self.Plugin_Name} results for query {self.Input}."
NL_Joined_Dump_Types = "\n- ".join(self.Dump_Types)
self.Ticket_Text = f"Results were identified for the search {self.Input} performed by the Scrummage plugin {self.Plugin_Name}.\nThe following types of sensitive data were found:\n- {NL_Joined_Dump_Types}. Please ensure these results do not pose a threat to your organisation, and take the appropriate action necessary if they pose a security risk.\n\nResult data can be found in the following output files:\n- {Text_Complete_Files}."
else:
self.Title = f"Data for input: {self.Input}, found by Scrummage plugin {self.Plugin_Name}.\nAll data is stored in the files:\n- {Text_Complete_Files}."
self.Ticket_Subject = f"Scrummage {self.Plugin_Name} results for query {self.Input}."
self.Ticket_Text = f"Results were identified for the search {self.Input} performed by the Scrummage plugin {self.Plugin_Name}. Please ensure these results do not pose a threat to your organisation, and take the appropriate action necessary if they pose a security risk.\n\nResult data can be found in the following output files:\n- {Text_Complete_Files}."
except:
logging.warning(
f"{Common.Date()} - General Library - Error setting unique variables."
)
logging.info(
f"{Common.Date()} - General Library - Adding item to Scrummage database and other configured outputs."
)
Connector_Object = Common.Configuration(Output=True)
CSV_File = Common.CSV_Output(Connector_Object, DB_Title,
self.Plugin_Name, self.Domain, Link,
self.Result_Type,
", ".join(Complete_File_List),
self.Task_ID, Directory_Plugin_Name)
DOCX_File = Common.DOCX_Output(Connector_Object, DB_Title,
self.Plugin_Name, self.Domain, Link,
self.Result_Type,
"\n".join(Complete_File_List),
self.Task_ID, Directory_Plugin_Name)
if CSV_File:
Complete_File_List.append(CSV_File)
if DOCX_File:
Complete_File_List.append(DOCX_File)
Relative_File_List = []
for File in Complete_File_List:
Relative_File = File.replace(
os.path.dirname(os.path.realpath('__file__')), "")
Relative_File_List.append(Relative_File)
Automated_Screenshots = Common.Configuration(
Core=True).Load_Configuration(
Object="web_scraping",
Details_to_Load=["risk_level", "automated_screenshots"])
if Automated_Screenshots[1]:
File_Dir = os.path.dirname(os.path.realpath('__file__'))
Screenshot_Path = Screenshot(
File_Dir, Internally_Requested=True,
Screenshot_Link=Link).Grab_Screenshot()
Common.Main_Database_Insert(Connector_Object,
DB_Title,
self.Plugin_Name,
self.Domain,
Link,
self.Result_Type,
", ".join(Relative_File_List),
self.Task_ID,
Screenshot_Path=Screenshot_Path)
else:
Common.Main_Database_Insert(Connector_Object, DB_Title,
self.Plugin_Name, self.Domain,
Link, self.Result_Type,
", ".join(Relative_File_List),
self.Task_ID)
Common.Elasticsearch_Main(Connector_Object, DB_Title,
self.Plugin_Name, self.Domain, Link,
self.Result_Type,
", ".join(Complete_File_List),
self.Task_ID, self.Concat_Plugin_Name)
Common.Defect_Dojo_Output(Connector_Object, DB_Title,
self.Ticket_Text)
Common.Scumblr_Main(Connector_Object, self.Input, DB_Title,
self.Title)
Common.RTIR_Main(Connector_Object, self.Ticket_Subject,
self.Ticket_Text)
Common.JIRA_Main(Connector_Object, self.Ticket_Subject,
self.Ticket_Text)
Common.Email_Main(Connector_Object, self.Ticket_Subject,
self.Ticket_Text)
Common.Slack_Main(Connector_Object, self.Ticket_Text)
except Exception as e:
logging.warning(
f"{Common.Date()} - General Library - Error handling outputs.")