def __init__(self, ioc, type, config, queues):
self.config = config
self.module_name = __name__.split(".")[-1]
self.types = ["MD5", "SHA256"]
self.search_method = "Onpremises"
self.description = "Search IOC in CuckooSandbox database"
self.author = "Conix"
self.creation_date = "02-03-2017"
self.type = type
self.ioc = ioc
self.queues = queues
self.verbose = "GET"
self.headers = self.config["user_agent"]
self.proxy = self.config["proxy_host"]
length = len(self.config['cuckoosandbox_api_url'])
if length != len(self.config['cuckoosandbox_web_url']) \
and length <= 0:
mod.display(
self.module_name, self.ioc, "ERROR",
"Cuckoosandbox fields in btg.cfg are missfilled, checkout commentaries."
)
return None
for indice in range(len(self.config['cuckoosandbox_api_url'])):
api_url = self.config['cuckoosandbox_api_url'][indice]
web_url = self.config['cuckoosandbox_web_url'][indice]
self.search(api_url, web_url, indice)
def search(self, api_url, web_url, indice):
mod.display(self.module_name, "", "INFO", "Searching...")
if ("cuckoosandbox_api_url" in self.config
and "user_agent" in self.config and "proxy_host" in self.config
and "requests_timeout" in self.config):
if self.type in ["MD5"]:
url = "%s/files/view/md5/%s" % (api_url, self.ioc)
elif self.type in ["SHA256"]:
url = "%s/files/view/sha256/%s" % (api_url, self.ioc)
request = {
'url': url,
'headers': self.headers,
'module': self.module_name,
'ioc': self.ioc,
'verbose': self.verbose,
'proxy': self.proxy,
'server_id': indice
}
json_request = json.dumps(request)
store_request(self.queues, json_request)
else:
mod.display(
self.module, self.ioc, "ERROR",
"Check if you have filled cuckoosandbox fields in btg.cfg")
def search(self):
mod.display(self.module_name, "", "INFO", "Searching...")
if "malekal_local" in self.config:
if self.config["malekal_local"]:
self.localSearch()
if "malekal_remote" in self.config:
if self.config["malekal_remote"]:
self.remoteSearch()
def search(self):
mod.display(self.module_name, self.ioc, "INFO",
"Search in MalwareConfig ...")
if self.type in ["MD5", "SHA256"]:
self.webpage_crawling()
else:
self.cache_search()
def cleanups_lock_cache(real_path):
for file in listdir(real_path):
file_path = "%s%s/" % (real_path, file)
if file.endswith(".lock"):
mod.display("MAIN",
message_type="DEBUG",
string="Delete locked cache file: %s" %
file_path[:-1])
remove(file_path[:-1])
else:
if path.isdir(file_path):
Utils.cleanups_lock_cache(file_path)
def createLoggingFolder():
if not isdir(config["log_folder"]):
try:
makedirs(config["log_folder"])
except:
mod.display(
"MAIN",
message_type="FATAL_ERROR",
string="Unable to create %s directory. (Permission denied)"
% config["log_folder"])
sys.exit()
chmod(config["log_folder"], 0o777)
async def fetch_post(url, session, headers, proxy, data, module, ioc, timeout,
auth, server_id, verify):
try:
async with session.post(url, data=data, headers=headers,
proxy=proxy, timeout=timeout,
auth=auth, ssl=verify) as response:
return await response.text(), response.status, module, ioc, server_id
except:
mod.display(module,
ioc,
message_type="ERROR",
string="Failed to connect to %s" % (url))
def Search(self):
mod.display(self.module_name, "", "INFO",
"Search in Alienvault OTX ...")
try:
if "otx_api_keys" in self.config:
otx = OTXv2(self.config["otx_api_keys"])
if self.type == "IPv4":
indicator = IndicatorTypes.IPv4
if self.type == "IPv6":
indicator = IndicatorTypes.IPv6
if self.type == "domain":
indicator = IndicatorTypes.DOMAIN
if self.type == "URL":
indicator = IndicatorTypes.URL
if self.type == "MD5":
indicator = IndicatorTypes.FILE_HASH_MD5
if self.type == "SHA1":
indicator = IndicatorTypes.FILE_HASH_SHA1
if self.type == "SHA256":
indicator = IndicatorTypes.FILE_HASH_SHA256
result = otx.get_indicator_details_full(indicator, self.ioc)
else:
mod.display(
self.module_name,
self.ioc,
message_type="ERROR",
string=
"Please check if you have otx_api_keys field in btg.cfg")
return None
except:
mod.display(
self.module_name, self.ioc, "ERROR",
"Could not perform the request, either you did not fill the otx_api_keys field or the key maximum request is reached"
)
return None
try:
if self.ioc == str(result["general"]["indicator"]):
_id = str(result["general"]["pulse_info"]["pulses"][0]["id"])
tags = ""
for tag in result["general"]["pulse_info"]["pulses"][0][
"tags"]:
tags = tags + "%s " % tag
mod.display(
self.module_name, self.ioc, "FOUND",
"Tags: %s| https://otx.alienvault.com/pulse/%s/" %
(tags, _id))
except:
mod.display(self.module_name,
self.ioc,
message_type="NOT_FOUND",
string="Nothing found in OTX")
return None
def createModuleFolder(self):
if not isdir(self.config["temporary_cache_path"]):
try:
makedirs(self.config["temporary_cache_path"])
except:
mod.display("%s.cache" % self.module_name,
"FATAL_ERROR",
"Unable to create %s directory. (Permission denied)" % self.config["temporary_cache_path"])
sys.exit()
chmod(self.config["temporary_cache_path"], 0o770)
if not isdir(self.temp_folder):
mkdir(self.temp_folder)
chmod(self.temp_folder, 0o770)
def module_worker_response(response_text, response_status, module,
ioc, server_id):
"""
Load modules in python instance to treat the response
"""
obj = importlib.import_module("BTG.modules."+module)
try:
obj.response_handler(response_text, response_status,
module, ioc, server_id)
except:
mod.display("worker_tasks",
"ERROR",
"Something went wrong when worker try to load response_handler from %s" % (module))
def search(self):
mod.display(self.module_name, "", "INFO", "Searching...")
self.url = "https://iris-h.services/api/search?hash=" + self.ioc
request = {'url': self.url,
'headers': self.headers,
'module': self.module_name,
'ioc': self.ioc,
'verbose': self.verbose,
'proxy': self.proxy
}
json_request = json.dumps(request)
store_request(self.queues, json_request)
def search(self):
mod.display(self.module_name, "", "INFO", "Searching...")
url = "http://www.nothink.org/blacklist/"
paths = [
"blacklist_snmp_year.txt",
"blacklist_ssh_year.txt",
"blacklist_telnet_year.txt"
]
for path in paths:
try:
content = Cache(self.module_name, url, path, self.search_method).content
except NameError as e:
mod.display(self.module_name,
self.ioc,
"ERROR",
e)
return None
for line in content.split("\n"):
if self.ioc in line:
mod.display(self.module_name,
self.ioc,
"FOUND",
"%s%s" % (url, path))
return None
mod.display(self.module_name,
self.ioc,
"NOT_FOUND",
"Nothing found in nothink feeds")
def response_handler(response_text,
response_status,
module,
ioc,
server_id=None):
if response_status == 200:
try:
json_content = json.loads(response_text)
except:
mod.display(module, ioc, "ERROR",
"VirusTotal json_response was not readable.")
return None
if "positives" in json_content and json_content["positives"] > 0:
mod.display(
module, ioc, "FOUND", "Score: %d/%d | %s" %
(json_content["positives"], json_content["total"],
json_content["permalink"]))
else:
mod.display(module, ioc, "NOT_FOUND",
"Nothing found in Virustotal")
else:
mod.display(module, ioc, "ERROR",
"VirusTotal response.code_status : %d" % (response_status))
return None
def search(self):
mod.display(self.module_name, "", "INFO", "Searching...")
url = "https://www.spamhaus.org/drop/"
paths = [
"drop.txt",
"edrop.txt",
"dropv6.txt",
]
for path in paths:
try:
content = Cache(self.module_name, url, path,
self.search_method).content
except NameError as e:
mod.display(self.module_name, self.ioc, "ERROR", e)
return None
for line in content.split("\n"):
try:
if line[0] != ';':
if IPAddress(self.ioc) in IPNetwork(
line.split(" ")[0]):
mod.display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
except:
pass
mod.display(self.module_name, self.ioc, "NOT_FOUND",
"Nothing found in SpamHaus feeds")
def response_handler(response_text, response_status, module, ioc, server_id):
web_url = cfg['misp_url'][server_id]
if response_status == 200:
try:
json_response = json.loads(response_text)
except:
mod.display(module,
ioc,
message_type="ERROR",
string="Misp json_response was not readable.")
return None
if "Attribute" in json_response["response"]:
displayed = []
for attr in json_response["response"]["Attribute"]:
event_id = attr["event_id"]
if event_id not in displayed:
displayed.append(event_id)
mod.display(module,
ioc,
"FOUND",
"Event: %sevents/view/%s" % (web_url,
event_id))
return None
mod.display(module,
ioc,
"NOT_FOUND",
"Nothing found in Misp:%s database" % (web_url))
return None
else:
mod.display(module,
ioc,
message_type="ERROR",
string="Misp connection status : %d" % (response_status))
def Search(self):
mod.display(self.module_name, "", "INFO", "Search in misp crawler...")
with requests.Session() as s:
try:
self.loginRequest(s)
allEvents = self.searchAttribute(s)
for event in allEvents:
if "misp_crawler_url" in self.config:
# TODO: And what is the response is empty, NOT_FOUND ?
mod.display(
self.module_name, self.ioc, "FOUND",
"Event: %s/events/view/%s" %
(self.config["misp_crawler_url"], event))
else:
mod.display(
self.module_name,
self.ioc,
message_type="ERROR",
string=
"Check if you have misp_crawler_url in btg.cfg")
except:
mod.display(
self.module_name,
self.ioc,
message_type="ERROR",
string=
"Could not perform the request, checkout btg.cfg at [%s]" %
(self.module_name))
def search(self):
mod.display(self.module_name, "", "INFO", "Searching...")
url = "https://www.dshield.org/feeds/"
paths = [
"suspiciousdomains_Low.txt",
"suspiciousdomains_Medium.txt",
"suspiciousdomains_High.txt"
]
for path in paths:
try:
content = Cache(self.module_name, url, path, self.search_method).content
except NameError as e:
mod.display(self.module_name,
self.ioc,
"ERROR",
e)
return None
for line in content.split("\n"):
try:
if line[0] != '#':
if line.lower() == self.ioc.lower():
mod.display(self.module_name,
self.ioc,
"FOUND",
"%s%s" % (url, path))
return None
except:
pass
mod.display(self.module_name,
self.ioc,
"NOT_FOUND",
"Nothing found in dhsield feeds")
def localSearch(self):
""" Search in local directory """
mod.display("%s_local" % self.module_name,
string="Browsing in local directory")
if "malekal_files_path" in self.config:
for root, dirs, files in os.walk(
self.config["malekal_files_path"]):
folder = os.path.basename(root)
for file in files:
if file == self.ioc:
mod.display(
"%s_local" % self.module_name, self.ioc, "FOUND",
"%s%s/%s" %
(self.config["malekal_files_path"], folder, file))
return None
mod.display("%s_local" % self.module_name,
self.ioc,
message_type="NOT_FOUND",
string="Nothing found in Malekal_local")
return None
else:
mod.display(
"%s_local" % self.module_name, self.ioc, "ERROR",
"Check if you have malekal_files_path field in btg.cfg ")
return None
def module_worker_request(module, argument, type, queues):
"""
Load modules in python instance to build url to request
"""
mod.display(string="Load: %s/%s.py" % (config["modules_folder"], module))
obj = importlib.import_module("BTG.modules."+module)
for c in dir(obj):
if module+"_enabled" in config:
if module == c.lower() and config[module+"_enabled"]:
getattr(obj, c)(argument, type, config, queues)
else:
mod.display("worker_tasks",
"INFO",
"Module : %s -- not configured" % (module))
def search(self):
mod.display(self.module_name, "", "INFO", "Search in VirusTotal ...")
try:
if "virustotal_api_keys" in self.config:
try:
self.key = random.Random(self.ioc).choice(
self.config["virustotal_api_keys"])
except:
mod.display(
self.module_name,
message_type="ERROR",
string=
"Check if you have filled virustotal_api_keys in btg.cfg"
)
return None
else:
mod.display(
self.module_name,
message_type="ERROR",
string=
"Check if you have virustotal_api_keys field in btg.cfg")
return None
except:
mod.display(self.module_name, self.ioc, "ERROR",
"Please provide your authkey.")
return None
if self.type in ["URL", "domain", "IPv4"]:
request = self.searchURL()
else:
request = self.searchReport()
store_request(self.queues, request)
def search(self):
mod.display(self.module_name, "", "INFO", "Searching...")
url = "https://torstatus.blutmagie.de/"
paths = [
"ip_list_all.php/Tor_ip_list_ALL.csv",
"query_export.php/Tor_query_EXPORT.csv",
"ip_list_exit.php/Tor_ip_list_EXIT.csv"
]
for path in paths:
try:
content = Cache(self.module_name, url, path, self.search_method).content
except NameError as e:
mod.display(self.module_name,
self.ioc,
"ERROR",
e)
return None
if self.ioc in content:
mod.display(self.module_name,
self.ioc,
"FOUND",
"%s%s" % (url, path))
return None
mod.display(self.module_name,
self.ioc,
"NOT_FOUND",
"Nothing found in TorIps feeds")
async def fetch_get(url, session, headers, proxy, module, ioc,
timeout, auth, server_id, verify):
"""
code from aiohttp.readthedocs.io
"""
try:
async with session.get(url, headers=headers, proxy=proxy,
timeout=timeout,
auth=auth, ssl=verify) as response:
return await response.text(), response.status, module, ioc, server_id
except:
mod.display(module,
ioc,
message_type="ERROR",
string="Failed to connect to %s, server was probably too slow and request has been dropped out" % (url))
async def bound_fetch(sem, session, request, timeout):
url, module_name, ioc, verbose, headers, proxy, auth, server_id, verify = filler(request)
if verbose == "GET":
# Getter function with semaphore.
async with sem:
return await fetch_get(url, session, headers, proxy, module_name,
ioc, timeout, auth, server_id, verify)
elif verbose == "POST":
data = request['data']
# Getter function with semaphore.
async with sem:
return await fetch_post(url, session, headers, proxy, data, module_name,
ioc, timeout, auth, server_id, verify)
else:
mod.display(module_name,
ioc,
message_type="ERROR",
string="Associated HTTP verbose for %s, is neither GET nor POST" % (url))
def __init__(self, args, modules):
redis_host, redis_port, redis_password = init_redis()
try:
conn = redis.StrictRedis(host=redis_host,
port=redis_port,
password=redis_password)
except:
mod.display("MAIN",
message_type="FATAL_ERROR",
string="Cannot establish connection with Redis")
sys.exit()
global observable_list
queues = [working_queue, request_queue]
if args.file == "False":
observable_list = args.observables
else:
observable_list = []
for file in args.observables:
with open(file, "r") as f1:
try:
observable_list = list(
set(observable_list +
f1.read().strip().splitlines()))
except:
mod.display(
"MAIN",
message_type="FATAL_ERROR",
string="Something went wrong with the argument file"
)
finally:
f1.close()
for argument in observable_list:
type = self.checkType(argument.lower())
if "split_observable" in config and config["split_observable"]:
if type == "URL" or type == "domain":
self.extend_IOC(argument.lower(), observable_list)
matching_list = Utils.gen_matching_type_modules_list(modules, type)
cluster.add_cluster(argument.lower(), matching_list, dictname,
conn)
self.run(argument.lower(), type, matching_list, queues)
print("Every IOCs have been enqueued, BTG is processing ...\n")
def subprocess_launcher():
"""
Subprocess loop to launch rq-worker
"""
processes = []
max_worker = number_of_worker()
worker_path = dirname(__file__) + '/lib/run_worker.py '
worker_params = '%s' % (working_queue)
worker_call = 'python3 ' + worker_path + worker_params
poller_path = dirname(__file__) + '/lib/poller.py '
poller_params = '%s %s' % (working_queue, request_queue)
poller_call = 'python3 ' + poller_path + poller_params
try:
for i in range(max_worker):
processes.append(
subprocess.Popen([worker_call],
shell=True,
preexec_fn=setsid).pid)
processes.append(
subprocess.Popen([poller_call], shell=True,
preexec_fn=setsid).pid)
except:
mod.display(
"MAIN",
message_type="FATAL_ERROR",
string="Could not launch workers and/or poller subprocesses")
sys.exit()
supervisor_path = dirname(__file__) + '/lib/hypervisor.py '
supervisor_params = '%d %s %s' % (getpid(), fp, working_queue)
for process in processes:
supervisor_params += ' ' + str(process)
supervisor_call = 'python3 ' + supervisor_path + supervisor_params
try:
processes.append(
subprocess.Popen([supervisor_call],
shell=True,
preexec_fn=setsid).pid)
except:
mod.display("MAIN",
message_type="FATAL_ERROR",
string="Could not launch supervisor subprocess")
sys.exit()
return processes
def vxstream_api(self):
"""
VXstream API Connection
"""
if 'vxstream_api_keys' in self.config:
try:
self.headers['api-key'] = random.Random(self.ioc).choice(
self.config['vxstream_api_keys'])
except:
mod.display(
self.module_name, self.ioc, "ERROR",
"Check if you have filled vxstream_api_keys_secret in btg.cfg"
)
return None
else:
mod.display(
self.module_name, self.ioc, "ERROR",
"Check if you have vxstream_api_keys_secret field in btg.cfg")
return None
if self.type in ["MD5", "SHA1", "SHA256"]:
self.url = "https://www.hybrid-analysis.com/api/v2/search/hash"
self.data = "hash=" + self.ioc
else:
self.url = "https://www.hybrid-analysis.com/api/v2/search/terms"
if self.type in ["IPv4", "IPv6"]:
self.data = "host=" + self.ioc
elif self.type == "URL":
self.data = "url=" + self.ioc
else:
self.data = "domain=" + self.ioc
request = {
'url': self.url,
'headers': self.headers,
'data': self.data,
'module': self.module_name,
'ioc': self.ioc,
'verbose': self.verbose,
'proxy': self.proxy
}
json_request = json.dumps(request)
store_request(self.queues, json_request)
def show_up_errors(start_time, end_time, modules):
"""
Count errors encountered during execution
"""
enabled_list = Utils.gen_enabled_modules_list(modules)
dict_list = []
for module in enabled_list:
dict_list.append({"module_name": module, "nb_error": 0})
log_error_file = config["log_folder"] + config["log_error_file"]
try:
with open(log_error_file, "r+") as f:
try:
lines = f.read().strip().splitlines()
except:
mod.display(
"MAIN", "FATAL_ERROR",
"Could not read %s, checkout your btg.cfg." %
(log_error_file))
sys.exit()
finally:
f.close()
except FileNotFoundError:
return None
except:
mod.display(
"MAIN", "FATAL_ERROR",
"Could not open %s, checkout your btg.cfg." % (log_error_file))
sys.exit()
regex = re.compile("(?<=\[).*?(?=\])")
start_time = start_time.strftime('%d-%m-%Y %H:%M:%S')
end_time = end_time.strftime('%d-%m-%Y %H:%M:%S')
for line in lines:
match = regex.findall(line)
log_time = match[0]
log_module = match[1]
if log_time >= start_time and log_time <= end_time:
for dict in dict_list:
tmp = "%s%s%s" % (colors.MODULE, dict['module_name'],
colors.NORMAL)
if log_module == tmp:
dict["nb_error"] = dict["nb_error"] + 1
return dict_list
def store_request(queues, request):
"""
Pushing request:(url, module_name) into a redis_list
"""
queue_2 = queues[1]
redis_host, redis_port, redis_password = init_redis()
try:
r = redis.StrictRedis(host=redis_host, port=redis_port,
password=redis_password, db=0)
except:
mod.display("ASYNC_HTTP",
"ERROR",
"Could not establish connection with Redis in func store_request")
try:
r.rpush(queue_2, request)
except:
mod.display("ASYNC_HTTP",
"ERROR",
"Cannot push request: %s to Redis on queue: %s" % (request[0], queue_2))
def pollout_requests(queue_2, nb_to_do):
redis_host, redis_port, redis_password = init_redis()
try:
r = redis.StrictRedis(host=redis_host, port=redis_port,
password=redis_password, db=0)
except:
mod.display("ASYNC_HTTP",
"ERROR",
"Cannot establish connection with Redis in func pollout_requests")
requests = []
while len(requests) < nb_to_do:
request = r.lpop(queue_2)
if request is None:
break
else:
request = parse_redis_string(request)
if request is not None:
requests.append(request)
return requests
def Search(self, misp_url, misp_key, indice):
mod.display(self.module_name, "", "INFO", "Search in misp...")
url = '%sattributes/restSearch/json' % (misp_url)
self.headers['Authorization'] = misp_key
payload = {'value': self.ioc, 'searchall': 1}
data = json.dumps(payload)
request = {'url': url,
'headers': self.headers,
'data': data,
'module': self.module_name,
'ioc': self.ioc,
'verbose': self.verbose,
'proxy': self.proxy,
'verify': self.verify,
'server_id': indice
}
json_request = json.dumps(request)
store_request(self.queues, json_request)