def process_iocs(results):
"""Return data formatted for Splunk from urlscan.io."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_url(provided_ioc)
if provided_ioc == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855":
splunk_table.append({"no data": provided_ioc})
continue
if validators.domain(provided_ioc) or validators.ipv4(provided_ioc) or \
validators.sha256(provided_ioc) or "certstream-suspicious" in provided_ioc:
ioc_dicts = query_urlscan(session, provided_ioc)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for ioc_dict in ioc_dicts:
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def process_iocs(results):
"""Return data formatted for Splunk from CyberCrime Tracker."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = provided_ioc.replace("[.]", ".")
provided_ioc = provided_ioc.replace("[d]", ".")
provided_ioc = provided_ioc.replace("[D]", ".")
if validators.domain(provided_ioc) or validators.ipv4(provided_ioc):
cct_dicts = query_cct(provided_ioc, session)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for cct_dict in cct_dicts:
splunk_table.append(cct_dict)
session.close()
return splunk_table
def process_iocs(results):
"""Return data formatted for Splunk from crt.sh."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
elif sys.argv[1] != "subdomain" and sys.argv[1] != "wildcard":
if len(sys.argv) > 1:
provided_iocs = sys.argv[1:]
elif sys.argv[1] == "subdomain" or sys.argv[1] == "wildcard":
if len(sys.argv) > 2:
provided_iocs = sys.argv[2:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_url(provided_ioc)
if validators.domain(provided_ioc) or validators.ipv4(provided_ioc):
crt_dicts = query_crtsh(provided_ioc, session)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for crt_dict in crt_dicts:
splunk_table.append(crt_dict)
session.close()
return splunk_table
def process_iocs(results):
"""Return data formatted for Splunk from Malshare."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
api_key = commons.get_apikey("malshare")
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_string(provided_ioc)
provided_ioc = provided_ioc.lower()
if validators.ipv4(provided_ioc) or validators.domain(provided_ioc) or \
re.match("^[a-f\d]{32}$", provided_ioc) or re.match("^[a-f\d]{64}$", provided_ioc):
pass
else:
splunk_table.append({"invalid": provided_ioc})
continue
ioc_dicts = query_malshare(provided_ioc, api_key, session)
for ioc_dict in ioc_dicts:
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def get_project():
"""Download the project to /tmp"""
session = commons.create_session()
project = "https://github.com/neonprimetime/PhishingKitTracker/archive/master.zip"
resp = session.get(project, timeout=180)
if not (resp.status_code == 200 and resp.content != ""):
return
with open("/tmp/master.zip", "wb") as repo:
repo.write(resp.content)
repo_zip = zipfile.ZipFile("/tmp/master.zip", "r")
repo_zip.extractall("/tmp/")
repo_zip.close()
# Remove current files
for csv in glob.glob(
"/{}/etc/apps/OSweep/lookups/2*_PhishingKitTracker.csv".format(
os.environ["SPLUNK_HOME"])):
os.remove(csv)
# Add new files
for csv in glob.glob(
"/tmp/PhishingKitTracker-master/2*_PhishingKitTracker.csv"):
shutil.move(
csv,
"/{}/etc/apps/OSweep/lookups".format(os.environ["SPLUNK_HOME"]))
os.remove("/tmp/master.zip")
shutil.rmtree("/tmp/PhishingKitTracker-master")
return
def process_iocs(results):
"""Return data formatted for Splunk from psbdmp."""
if sys.argv[1] == "search" or sys.argv[1] == "dump":
endpoint = sys.argv[1]
provided_iocs = sys.argv[2:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_string(provided_ioc)
if endpoint == "search":
psbdmp_dicts = psbdmp_search(provided_ioc, session)
elif endpoint == "dump":
psbdmp_dicts = psbdmp_dump(provided_ioc, session)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for psbdmp_dict in psbdmp_dicts:
splunk_table.append(psbdmp_dict)
session.close()
return splunk_table
def get_feed():
"""Return the latest report summaries from the feed."""
session = commons.create_session()
data_feed = check_project(session)
if data_feed == None:
return
return data_feed
def get_feed():
"""Return the latest report summaries from the feed."""
session = commons.create_session()
tags = query_list(session)
if tags == None:
return
return query_tags(tags, session)
def get_modules():
"""Return Phishing Catcher modules."""
session = commons.create_session()
suspicious = request_module(session, "/phishing_catcher_suspicious.yaml")
confusables = request_module(session, "/phishing_catcher_confusables.py")
session.close()
if suspicious == None or confusables == None:
return
return suspicious, confusables
def get_feed():
"""Return the latest report summaries from the feed."""
session = commons.create_session()
api_key = commons.get_apikey("greynoise")
tags = query_list(session)
if tags == None:
return
if api_key != None:
session.params = {"key": api_key}
return query_tags(tags, session)
def get_feed():
"""Return OSINT data feed."""
api = "http://cybercrime-tracker.net/all.php"
session = commons.create_session()
resp = session.get(api)
session.close()
if resp.status_code == 200 and resp.text != "":
data = resp.text.splitlines()
header = "url"
data_feed = []
for line in data:
data_feed.append({"url": line})
return data_feed
return
def process_iocs(results):
"""Return data formatted for Splunk from URLhaus."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
empty_files = [
"d41d8cd98f00b204e9800998ecf8427e",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
]
urlhaus_match = re.compile(r"^h..ps?:\/\/urlhaus\.abuse\.ch")
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_string(provided_ioc)
if provided_ioc in empty_files:
splunk_table.append({"invalid": provided_ioc})
continue
if urlhaus_match.match(provided_ioc):
splunk_table.append({"invalid": provided_ioc})
continue
if validators.domain(provided_ioc) or validators.ipv4(provided_ioc):
ioc_type = "host"
elif validators.url(provided_ioc):
ioc_type = "url"
elif re.match("^[a-f\d]{32}$", provided_ioc) or re.match(
"^[a-f\d]{64}$", provided_ioc):
ioc_type = "payload"
else:
splunk_table.append({"invalid": provided_ioc})
continue
ioc_dicts = query_urlhaus(session, provided_ioc, ioc_type)
for ioc_dict in ioc_dicts:
ioc_dict = commons.lower_keys(ioc_dict)
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def process_iocs(results):
"""Return data formatted for Splunk from urlscan.io."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
elif sys.argv[1] in usfs.queries.keys():
if len(sys.argv[1:]) < 3:
return [{
"error":
"3 positional args needed. {} given.".format(
str(len(sys.argv[1:])))
}]
provided_iocs = sys.argv[3:]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_string(provided_ioc)
if provided_ioc == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855":
splunk_table.append({"no data": provided_ioc})
continue
if provided_ioc.lower() in usfs.extensions.keys():
ioc_dicts = query_urlscan_file(session, provided_ioc)
elif validators.domain(provided_ioc) or validators.ipv4(provided_ioc) or \
validators.sha256(provided_ioc) or "certstream-suspicious" in provided_ioc:
ioc_dicts = query_urlscan(session, provided_ioc)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for ioc_dict in ioc_dicts:
if "ip" not in ioc_dict:
splunk_table.append({"no data": provided_ioc})
continue
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def process_iocs(results):
"""Return data formatted for Splunk from Cymon."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in provided_iocs:
provided_ioc = provided_ioc.replace("[.]", ".")
provided_ioc = provided_ioc.replace("[d]", ".")
provided_ioc = provided_ioc.replace("[D]", ".")
if validators.ipv4(provided_ioc):
ioc_type = "ip"
elif validators.domain(provided_ioc):
ioc_type = "domain"
elif validators.md5(provided_ioc):
ioc_type = "md5"
elif validators.sha256(provided_ioc):
ioc_type = "sha256"
else:
splunk_table.append({"invalid": provided_ioc})
continue
ioc_dicts = query_cymon(ioc_type, session, provided_ioc)
if isinstance(ioc_dicts, dict):
splunk_table.append(ioc_dicts)
continue
for ioc_dict in ioc_dicts:
ioc_dict = commons.lower_keys(ioc_dict)
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def process_iocs(results):
"""Return data formatted for Splunk from ThreatCrowd."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = provided_ioc.replace("[.]", ".")
provided_ioc = provided_ioc.replace("[d]", ".")
provided_ioc = provided_ioc.replace("[D]", ".")
if validators.ipv4(provided_ioc):
ioc_type = "ip"
elif validators.domain(provided_ioc):
ioc_type = "domain"
elif validators.email(provided_ioc):
ioc_type = "email"
if validators.ipv4(provided_ioc) or validators.domain(provided_ioc):
ioc_dicts = query_threatcrowd(provided_ioc, ioc_type, session)
elif validators.email(provided_ioc):
ioc_dicts = query_threatcrowd(provided_ioc, ioc_type, session)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for ioc_dict in ioc_dicts:
splunk_table.append(ioc_dict)
if len(provided_iocs) > 1:
sleep(10)
session.close()
return splunk_table
def process_iocs(results):
"""Return data formatted for Splunk from Hybrid-Analysis."""
params = [
'authentihash', 'av_detect', 'context', 'country', 'domain', 'env_id',
'filename', 'filetype_desc', 'filetype', 'hash', 'host', 'imp_hash',
'port', 'similar_to', 'ssdeep', 'tag', 'url', 'verdict', 'vx_family'
]
if results != None:
provided_iocs = [y for x in results for y in x.values()]
elif sys.argv[1] == "terms" and sys.argv[2] in params:
if len(sys.argv) > 2:
endpoint = sys.argv[1]
param = sys.argv[2]
provided_iocs = sys.argv[3:]
elif sys.argv[1] == "hash" and sys.argv[2] == "hash":
if len(sys.argv) > 2:
endpoint = sys.argv[1]
param = sys.argv[2]
provided_iocs = sys.argv[3:]
session = commons.create_session()
api_domain = commons.get_apidomain("hybrid-analysis")
api_key = commons.get_apikey("hybrid-analysis")
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_string(provided_ioc)
provided_ioc = provided_ioc.lower()
ioc_dicts = query_hybridanalysis(endpoint, param, provided_ioc,
api_domain, api_key, session)
for ioc_dict in ioc_dicts:
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def get_feed():
"""Return the latest report summaries from the feed."""
api = "https://ransomwaretracker.abuse.ch/feeds/csv/"
session = commons.create_session()
resp = session.get(api, timeout=180)
if resp.status_code == 200 and resp.text != "":
data = resp.text.splitlines()
data = data[8:-1]
data[0] = data[0][2:]
header = data[0].split(",")
data_feed = []
for line in data[1:]:
line = line.replace('","', "^^")
line = line.replace(",", " ")
line = line.replace("^^", ",")
line = line.replace('"', " ")
ransomware_data = line.split(",")
ransomware_dict = OrderedDict(zip(header, ransomware_data))
data_feed.append(ransomware_dict)
return data_feed
return
def process_iocs(results):
"""Return data formatted for Splunk from ThreatCrowd."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_string(provided_ioc)
provided_ioc = provided_ioc.lower()
if validators.ipv4(provided_ioc):
ioc_type = "ip"
elif validators.domain(provided_ioc):
ioc_type = "domain"
elif validators.email(provided_ioc):
ioc_type = "email"
elif re.match("^[a-f\d]{32}$", provided_ioc) or re.match(
"^[a-f\d]{64}$", provided_ioc):
ioc_type = "resource"
else:
splunk_table.append({"invalid": provided_ioc})
continue
ioc_dicts = query_threatcrowd(provided_ioc, ioc_type, session)
for ioc_dict in ioc_dicts:
splunk_table.append(ioc_dict)
if len(provided_iocs) > 1:
sleep(10)
session.close()
return splunk_table
def get_feed():
"""Return the latest report summaries from the feed."""
api = "https://urlhaus.abuse.ch/downloads"
session = commons.create_session()
resp = session.get("{}/csv/".format(api))
session.close()
if resp.status_code == 200 and resp.text != "":
data = resp.text.splitlines()
data = data[8:]
data[0] = data[0][2:]
header = data[0].split(",")
data_feed = []
for line in data[1:]:
line = line.replace('","', "^^")
line = line.replace(",", " ")
line = line.replace("^^", ",")
line = line.replace('"', "")
ransomware_data = line.split(",")
ransomware_dict = OrderedDict(zip(header, ransomware_data))
data_feed.append(ransomware_dict)
return data_feed
return
def process_iocs(results):
"""Return data formatted for Splunk from URLhaus."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
lookup_path = "/opt/splunk/etc/apps/osweep/lookups"
open_file = open("{}/urlhaus_url_feed.csv".format(lookup_path), "r")
contents = open_file.read().splitlines()
open_file.close()
header = contents[0].split(",")
global data_feed
data_feed = []
for line in contents:
line = line.split(",")
ioc_dict = OrderedDict(zip(header, line))
data_feed.append(ioc_dict)
global parser
parser = ParserHTML()
empty_files = [
"d41d8cd98f00b204e9800998ecf8427e",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
]
urlhaus_match = re.compile(r"^h..ps?:\/\/urlhaus\.abuse\.ch")
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_url(provided_ioc)
if provided_ioc in empty_files:
splunk_table.append({"invalid": provided_ioc})
continue
if urlhaus_match.match(provided_ioc):
splunk_table.append({"invalid": provided_ioc})
continue
if validators.url(provided_ioc) or validators.domain(provided_ioc) or \
validators.ipv4(provided_ioc):
analysis_dicts = get_analysis(provided_ioc)
if isinstance(analysis_dicts, dict):
splunk_table.append(analysis_dicts)
continue
ioc_dicts = get_payloads(analysis_dicts, session)
elif validators.md5(provided_ioc) or validators.sha256(provided_ioc):
ioc_dicts = get_urls(session, provided_ioc)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for ioc_dict in ioc_dicts:
ioc_dict = commons.lower_keys(ioc_dict)
splunk_table.append(ioc_dict)
time.sleep(1)
session.close()
return splunk_table