def module_worker(self, module, argument, type):
"""
Load modules in python instance
"""
display(string="Load: %s/%s.py" % (config.modules_folder, module))
exec("from modules.%s import %s" % (module, module.title()))
exec("event = %s('%s', '%s')" % (module.title(), argument, type))
def searchURL(self):
self.url = "http://www.virustotal.com/vtapi/v2/url/report"
parameters = {"resource": self.ioc, "apikey": self.key}
data = urllib.urlencode(parameters)
req = urllib2.Request(self.url, data)
while True:
try:
response = urllib2.urlopen(req).read()
json_content = loads(response)
break
except:
display(
self.module_name, self.ioc, "INFO",
"Virustotal json decode fail. Mabye: Blacklisted/Bad API key (Sleep for 10 seconds)."
)
sleep(randint(5, 10))
pass
try:
if json_content["positives"]:
display(
self.module_name, self.ioc, "FOUND", "Score: %s/%s | %s" %
(json_content["positives"], json_content["total"],
json_content["permalink"]))
except:
pass
def __init__(self, args):
# Import modules
if config.debug:
display(string="Load modules from %s" % config.modules_folder)
char = '/'
if system() == "Windows":
char = '\\'
all_files = [
f for f in listdir(config.modules_folder)
if isfile(join(config.modules_folder, f))
]
modules = []
for file in all_files:
if file[-3:] == ".py" and file[:-3] != "__init__":
modules.append(file[:-3])
jobs = []
# Start BTG process
i = 0
for argument in args:
i += 1
p = multiprocessing.Process(target=self.run,
args=(
argument,
modules,
))
while len(jobs) > config.max_process:
for job in jobs:
if not job.is_alive():
jobs.remove(job)
else:
sleep(3)
jobs.append(p)
p.start()
def loginRequest(self, s):
if "misp_crawler_url" in self.config and "user_agent" in self.config and "misp_crawler_verifycert" in self.config:
response = s.get("%s/users/login" %
self.config["misp_crawler_url"],
headers=self.config["user_agent"],
verify=self.config["misp_crawler_verifycert"])
token_key, token_fields = self.getTokens(response.text)
data = {
'_method': 'POST',
'data[_Token][key]': token_key,
'data[User][email]': self.config["misp_crawler_login"],
'data[User][password]': self.config["misp_crawler_password"],
'data[_Token][fields]': token_fields,
'data[_Token][unlocked]': ''
}
response = s.post("%s/users/login" %
self.config["misp_crawler_url"],
data=data,
headers=self.config["user_agent"],
verify=self.config["misp_crawler_verifycert"])
else:
display(
self.module_name,
message_type="ERROR",
string=
"Please check if you have misp_crawler_url, user_agent and misp_crawler_verifycert fields in config.ini"
)
def __init__(self, ioc, type):
if config.virustotal_enabled:
self.module_name = __name__.split(".")[1]
self.types = ["MD5", "SHA1", "SHA256", "URL", "IPv4", "domain"]
self.search_method = "Online"
self.description = "Search IOC in VirusTotal database"
self.author = "Conix"
self.creation_date = "13-09-2016"
self.type = type
self.ioc = ioc
if type in self.types and BTG.allowedToSearch(self.search_method):
if len(config.proxy_host["https"]) > 0:
proxy = urllib2.ProxyHandler(
{'https': config.proxy_host["https"]})
opener = urllib2.build_opener(proxy)
else:
opener = urllib2.build_opener()
urllib2.install_opener(opener)
try:
self.key = choice(config.virustotal_API_keys)
except:
display(self.module_name, self.ioc, "ERROR",
"Please provide your authorization key.")
return
display(self.module_name, self.ioc, "INFO", "Searching...")
if self.type in ["URL", "domain", "IPv4"]:
self.searchURL()
else:
self.searchReport()
def __init__(self, ioc, type, config):
self.config = config
self.module_name = __name__.split(".")[1]
if "malekal_local" in self.config and "malekal_remote" in self.config:
if self.config[
"malekal_local"] and not self.config["malekal_remote"]:
self.types = ["MD5"]
else:
self.types = [
"MD5", "SHA1", "SHA256", "SHA512", "URL", "IPv4", "IPv6",
"domain"
]
else:
display(
self.module_name,
message_type="ERROR",
string=
"Please check if you have malekal_local and malekal_remote fields in config.ini "
)
self.search_method = "Online"
self.description = "Search IOC in malekal database"
self.author = "Conix"
self.creation_date = "13-09-2016"
self.type = type
self.ioc = ioc
if type in self.types:
self.search()
def checkType(self, argument):
"""
Identify IOC type
"""
if validators.url(argument):
return "URL"
elif len(findall(r"^([a-fA-F\d]{32}$)", argument)):
return "MD5"
elif len(findall(r"^[0-9a-f]{5,40}$", argument)):
return "SHA1"
elif len(findall(r"^[A-Fa-f0-9]{64}$", argument)):
return "SHA256"
elif len(findall(r"^\w{128}$", argument)):
return "SHA512"
elif parse.is_valid_ipv4_address(argument):
return "IPv4"
elif parse.is_valid_ipv6_address(argument):
return "IPv6"
elif len(
findall(
r"^(?=.{4,255}$)([a-zA-Z0-9][a-zA-Z0-9-]{,61}[a-zA-Z0-9]\.)+[a-zA-Z0-9]{2,5}$",
argument)) or parse.is_valid_domain(argument):
return "domain"
else:
display("MAIN", argument, "ERROR", "Unable to retrieve IOC type")
return None
def Search(self):
display(self.module_name, self.ioc, "INFO", "Search in misp...")
try:
m = PyMISP(config.misp_url, config.misp_key,
config.misp_verifycert, 'json')
except Exception, e:
display(self.module_name, self.ioc, "ERROR", e)
return
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
if "malekal_local" in self.config:
if self.config["malekal_local"]:
self.localSearch()
if "malekal_remote" in self.config:
if self.config["malekal_remote"] and BTG.allowedToSearch(
self.search_method):
self.remoteSearch()
def module_worker(self, module, argument, type):
"""
Load modules in python instance
"""
display(string="Load: %s/%s.py" % (config["modules_folder"], module))
obj = importlib.import_module("modules." + module)
for c in dir(obj):
if module + "_enabled" in config:
if module == c.lower() and config[module + "_enabled"]:
getattr(obj, c)(argument, type, config)
class Misp:
def __init__(self, ioc, type):
if config.misp_enabled:
self.module_name = __name__.split(".")[1]
self.types = [
"MD5", "SHA1", "domain", "IPv4", "IPv6", "URL", "SHA256",
"SHA512"
]
self.search_method = "Offline"
self.description = "Search IOC in MISP database"
self.author = "Conix"
self.creation_date = "07-10-2016"
self.type = type
self.ioc = ioc
if type in self.types:
self.Search()
def Search(self):
display(self.module_name, self.ioc, "INFO", "Search in misp...")
try:
m = PyMISP(config.misp_url, config.misp_key,
config.misp_verifycert, 'json')
except Exception, e:
display(self.module_name, self.ioc, "ERROR", e)
return
result = m.search_all(self.ioc)
try:
for event in result["response"]:
tag_display = ""
try:
for tag in event["Event"]["Tag"]:
if tag["name"].split(
":")[0] in config.misp_tag_display:
if len(tag_display) == 0:
tag_display = "["
else:
tag_display = "%s|" % tag_display
tag_display = "%s%s" % (tag_display, tag["name"])
except:
pass
if len(tag_display) != 0:
tag_display = "%s]" % tag_display
display(self.module_name, self.ioc, "FOUND",
"%s Event: %s" % (tag_display, event["Event"]["id"]))
except:
try:
if result['message'] == "No matches":
pass
elif "Authentication failed" in result['message']:
display(__name__.split(".")[1],
message_type="ERROR",
string=result['message'])
except:
pass
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "http://mirror1.malwaredomains.com/files/"
paths = ["immortal_domains.txt"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
if line == self.ioc:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "https://palevotracker.abuse.ch/"
paths = ["blocklists.php?download=domainblocklist"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content:
if line == self.ioc:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "http://torstatus.blutmagie.de/"
paths = [
"ip_list_all.php/Tor_ip_list_ALL.csv",
"query_export.php/Tor_query_EXPORT.csv",
"ip_list_exit.php/Tor_ip_list_EXIT.csv"
]
for path in paths:
if self.ioc in Cache(self.module_name, url, path, self.search_method).content:
display(self.module_name, self.ioc, "FOUND", "%s%s"%(url, path))
def cleanups_lock_cache(real_path):
for file in listdir(real_path):
file_path = "%s%s/" % (real_path, file)
if file.endswith(".lock"):
display("MAIN",
message_type="DEBUG",
string="Delete locked cache file: %s" % file_path[:-1])
remove(file_path[:-1])
else:
if path.isdir(file_path):
cleanups_lock_cache(file_path)
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "http://vxvault.net/"
paths = ["URL_List.php"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
if self.ioc in line:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
return
def localSearch(self):
""" Search in local directory """
display("%s_local" % self.module_name,
string="Browsing in local directory")
for root, dirs, files in os.walk(config.malekal_files_path):
path = root.split('/')
folder = os.path.basename(root)
for file in files:
if file == self.ioc:
display(
"%s_local" % self.module_name, self.ioc, "FOUND",
"%s%s/%s" % (config.malekal_files_path, folder, file))
def searchReport(self):
self.url = "https://www.virustotal.com/vtapi/v2/file/report"
parameters = {"resource": self.ioc, "apikey": self.key}
data = urllib.urlencode(parameters)
req = urllib2.Request(self.url, data)
response = urllib2.urlopen(req)
json_content = loads(response.read())
try:
display(self.module_name, self.ioc, "FOUND",
json_content["permalink"])
except:
pass
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "https://www.malwaredomainlist.com/hostslist/"
paths = ["hosts.txt"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
if "127.0.0.1" in line:
if self.ioc == line.split(" ")[1].strip():
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "http://malwaredomains.lehigh.edu/files/"
paths = ["domains.txt"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
if line and line[0] != '#':
base = line.split("\t\t")[1]
if self.ioc == base.split("\t")[0]:
display(self.module_name, self.ioc, "FOUND",
"[%s] %s%s" % (base.split("\t")[1], url, path))
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "http://cybercrime-tracker.net/"
paths = ["all.php"]
if self.type == "URL":
self.ioc = self.ioc.split("//")[1]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
if self.ioc in line:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "https://sslbl.abuse.ch/blacklist/"
paths = ["sslblacklist.csv"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
if self.ioc in line:
infos = line.split(',')
display(self.module_name, self.ioc, "FOUND",
"%s | %s%s" % (infos[2], url, path))
return
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "https://openphish.com/"
paths = ["feed.txt"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
try:
midle = line.split("//")[-1].split("/")[0]
except:
midle = None
if self.type == "URL":
if self.ioc in line:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
return
elif self.type == "IPv4" and parse.is_valid_ipv4_address(
midle):
if self.ioc == midle:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
return
elif self.type == "domain" and parse.is_valid_domain(midle):
if midle == self.ioc:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
return
def remoteSearch(self):
"""
Search IOC with HTTP request
"""
display("%s_remote" % self.module_name,
self.ioc,
"INFO",
string="Browsing in remote http")
url = "http://malwaredb.malekal.com/index.php?"
if self.type in ["MD5", "SHA1", "SHA256", "SHA512"]:
base = "hash="
elif self.type in ["URL", "domain"]:
base = "url="
elif self.type in ["IPv4", "IPv6"]:
base = "domaine="
try:
if "user_agent" in self.config and "proxy_host" in self.config and "requests_timeout" in self.config:
page = get("%s%s%s" % (url, base, self.ioc),
headers=self.config["user_agent"],
proxies=self.config["proxy_host"],
timeout=self.config["requests_timeout"]).text
if len(findall("hash=([a-z0-9]{32})\"", page)) > 1:
display("%s_remote" % self.module_name, self.ioc, "FOUND",
"%s%s%s" % (url, base, self.ioc))
else:
display(
self.module_name,
message_type="ERROR",
string=
"Please check if you have user_agent, proxy_host and requests_timeout fields in config.ini "
)
except:
display("%s_remote" % self.module_name, self.ioc, "INFO",
"MalekalTimeout")
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "http://hosts-file.malwareteks.com/"
paths = ["hosts.txt"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
try:
if self.ioc == line.split("127.0.0.1")[1].strip():
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
except:
pass
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "http://www.nothink.org/blacklist/"
paths = [
"blacklist_snmp_year.txt", "blacklist_ssh_year.txt",
"blacklist_telnet_year.txt"
]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
if self.ioc in line:
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "https://ransomwaretracker.abuse.ch/feeds/"
paths = ["csv"]
content = Cache(self.module_name, url, paths[0],
self.search_method).content
for line in content.split("\n"):
try:
if self.ioc in line:
display(
self.module_name, self.ioc,
"FOUND", "%s | %s%s" % (line.split(",")[2].replace(
'"', '', 2), url, paths[0]))
except:
pass
def createModuleFolder(self):
if not isdir(config.temporary_cache_path):
try:
mkdir(config.temporary_cache_path)
except:
display(
"%s.cache" % self.module_name,
message_type="ERROR",
string="Unable to create %s directory. (Permission denied)"
% config.temporary_cache_path)
sys.exit()
chmod(config.temporary_cache_path, 0o777)
if not isdir(self.temp_folder):
mkdir(self.temp_folder)
chmod(self.temp_folder, 0o777)
class Otx:
def __init__(self, ioc, type, config):
self.config = config
self.module_name = __name__.split(".")[1]
self.types = ["MD5", "SHA1", "domain", "IPv4", "IPv6", "URL", "SHA256"]
self.search_method = "Online"
self.description = "Search IOC in Alienvault database"
self.author = "Hicham Megherbi"
self.creation_date = "13-04-2016"
self.type = type
self.ioc = ioc
if type in self.types:
self.Search()
def Search(self):
display(self.module_name, self.ioc, "INFO", "Search in Alienvault...")
try:
if "otx_api_keys" in self.config:
otx = OTXv2(self.config["otx_api_keys"])
if self.type == "IPv4":
indicator = IndicatorTypes.IPv4
if self.type == "IPv6":
indicator = IndicatorTypes.IPv6
if self.type == "domain":
indicator = IndicatorTypes.DOMAIN
if self.type == "URL":
indicator = IndicatorTypes.URL
if self.type == "MD5":
indicator = IndicatorTypes.FILE_HASH_MD5
if self.type == "SHA1":
indicator = IndicatorTypes.FILE_HASH_SHA1
if self.type == "SHA256":
indicator = IndicatorTypes.FILE_HASH_SHA256
result = otx.get_indicator_details_full(indicator, self.ioc)
else:
display(
self.module_name,
message_type="ERROR",
string=
"Please check if you have otx_api_keys field in config.ini"
)
except Exception, e:
display(self.module_name, self.ioc, "ERROR", e)
return
try:
if self.ioc == str(result["general"]["indicator"]):
_id = str(result["general"]["pulse_info"]["pulses"][0]["id"])
tags = ""
for tag in result["general"]["pulse_info"]["pulses"][0][
"tags"]:
tags = tags + "%s " % tag
display(
self.module_name, self.ioc, "FOUND",
"Tags: %s| https://otx.alienvault.com/pulse/%s/" %
(tags, _id))
except:
pass
def search(self):
display(self.module_name, self.ioc, "INFO", "Searching...")
url = "https://www.dshield.org/feeds/"
paths = [
"suspiciousdomains_Low.txt", "suspiciousdomains_Medium.txt",
"suspiciousdomains_High.txt"
]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
try:
if line[0] != '#':
if line.lower() == self.ioc.lower():
display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
except:
pass
