def process_iocs(results): """Return data formatted for Splunk from urlscan.io.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_url(provided_ioc) if provided_ioc == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855": splunk_table.append({"no data": provided_ioc}) continue if validators.domain(provided_ioc) or validators.ipv4(provided_ioc) or \ validators.sha256(provided_ioc) or "certstream-suspicious" in provided_ioc: ioc_dicts = query_urlscan(session, provided_ioc) else: splunk_table.append({"invalid": provided_ioc}) continue for ioc_dict in ioc_dicts: splunk_table.append(ioc_dict) session.close() return splunk_table
def process_iocs(results): """Return data formatted for Splunk from CyberCrime Tracker.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = provided_ioc.replace("[.]", ".") provided_ioc = provided_ioc.replace("[d]", ".") provided_ioc = provided_ioc.replace("[D]", ".") if validators.domain(provided_ioc) or validators.ipv4(provided_ioc): cct_dicts = query_cct(provided_ioc, session) else: splunk_table.append({"invalid": provided_ioc}) continue for cct_dict in cct_dicts: splunk_table.append(cct_dict) session.close() return splunk_table
def process_iocs(results): """Return data formatted for Splunk from crt.sh.""" if results != None: provided_iocs = [y for x in results for y in x.values()] elif sys.argv[1] != "subdomain" and sys.argv[1] != "wildcard": if len(sys.argv) > 1: provided_iocs = sys.argv[1:] elif sys.argv[1] == "subdomain" or sys.argv[1] == "wildcard": if len(sys.argv) > 2: provided_iocs = sys.argv[2:] session = commons.create_session() splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_url(provided_ioc) if validators.domain(provided_ioc) or validators.ipv4(provided_ioc): crt_dicts = query_crtsh(provided_ioc, session) else: splunk_table.append({"invalid": provided_ioc}) continue for crt_dict in crt_dicts: splunk_table.append(crt_dict) session.close() return splunk_table
def process_iocs(results): """Return data formatted for Splunk from Malshare.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() api_key = commons.get_apikey("malshare") splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_string(provided_ioc) provided_ioc = provided_ioc.lower() if validators.ipv4(provided_ioc) or validators.domain(provided_ioc) or \ re.match("^[a-f\d]{32}$", provided_ioc) or re.match("^[a-f\d]{64}$", provided_ioc): pass else: splunk_table.append({"invalid": provided_ioc}) continue ioc_dicts = query_malshare(provided_ioc, api_key, session) for ioc_dict in ioc_dicts: splunk_table.append(ioc_dict) session.close() return splunk_table
def get_project(): """Download the project to /tmp""" session = commons.create_session() project = "https://github.com/neonprimetime/PhishingKitTracker/archive/master.zip" resp = session.get(project, timeout=180) if not (resp.status_code == 200 and resp.content != ""): return with open("/tmp/master.zip", "wb") as repo: repo.write(resp.content) repo_zip = zipfile.ZipFile("/tmp/master.zip", "r") repo_zip.extractall("/tmp/") repo_zip.close() # Remove current files for csv in glob.glob( "/{}/etc/apps/OSweep/lookups/2*_PhishingKitTracker.csv".format( os.environ["SPLUNK_HOME"])): os.remove(csv) # Add new files for csv in glob.glob( "/tmp/PhishingKitTracker-master/2*_PhishingKitTracker.csv"): shutil.move( csv, "/{}/etc/apps/OSweep/lookups".format(os.environ["SPLUNK_HOME"])) os.remove("/tmp/master.zip") shutil.rmtree("/tmp/PhishingKitTracker-master") return
def process_iocs(results): """Return data formatted for Splunk from psbdmp.""" if sys.argv[1] == "search" or sys.argv[1] == "dump": endpoint = sys.argv[1] provided_iocs = sys.argv[2:] session = commons.create_session() splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_string(provided_ioc) if endpoint == "search": psbdmp_dicts = psbdmp_search(provided_ioc, session) elif endpoint == "dump": psbdmp_dicts = psbdmp_dump(provided_ioc, session) else: splunk_table.append({"invalid": provided_ioc}) continue for psbdmp_dict in psbdmp_dicts: splunk_table.append(psbdmp_dict) session.close() return splunk_table
def get_feed(): """Return the latest report summaries from the feed.""" session = commons.create_session() data_feed = check_project(session) if data_feed == None: return return data_feed
def get_feed(): """Return the latest report summaries from the feed.""" session = commons.create_session() tags = query_list(session) if tags == None: return return query_tags(tags, session)
def get_modules(): """Return Phishing Catcher modules.""" session = commons.create_session() suspicious = request_module(session, "/phishing_catcher_suspicious.yaml") confusables = request_module(session, "/phishing_catcher_confusables.py") session.close() if suspicious == None or confusables == None: return return suspicious, confusables
def get_feed(): """Return the latest report summaries from the feed.""" session = commons.create_session() api_key = commons.get_apikey("greynoise") tags = query_list(session) if tags == None: return if api_key != None: session.params = {"key": api_key} return query_tags(tags, session)
def get_feed(): """Return OSINT data feed.""" api = "http://cybercrime-tracker.net/all.php" session = commons.create_session() resp = session.get(api) session.close() if resp.status_code == 200 and resp.text != "": data = resp.text.splitlines() header = "url" data_feed = [] for line in data: data_feed.append({"url": line}) return data_feed return
def process_iocs(results): """Return data formatted for Splunk from URLhaus.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() empty_files = [ "d41d8cd98f00b204e9800998ecf8427e", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ] urlhaus_match = re.compile(r"^h..ps?:\/\/urlhaus\.abuse\.ch") splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_string(provided_ioc) if provided_ioc in empty_files: splunk_table.append({"invalid": provided_ioc}) continue if urlhaus_match.match(provided_ioc): splunk_table.append({"invalid": provided_ioc}) continue if validators.domain(provided_ioc) or validators.ipv4(provided_ioc): ioc_type = "host" elif validators.url(provided_ioc): ioc_type = "url" elif re.match("^[a-f\d]{32}$", provided_ioc) or re.match( "^[a-f\d]{64}$", provided_ioc): ioc_type = "payload" else: splunk_table.append({"invalid": provided_ioc}) continue ioc_dicts = query_urlhaus(session, provided_ioc, ioc_type) for ioc_dict in ioc_dicts: ioc_dict = commons.lower_keys(ioc_dict) splunk_table.append(ioc_dict) session.close() return splunk_table
def process_iocs(results): """Return data formatted for Splunk from urlscan.io.""" if results != None: provided_iocs = [y for x in results for y in x.values()] elif sys.argv[1] in usfs.queries.keys(): if len(sys.argv[1:]) < 3: return [{ "error": "3 positional args needed. {} given.".format( str(len(sys.argv[1:]))) }] provided_iocs = sys.argv[3:] else: provided_iocs = sys.argv[1:] session = commons.create_session() splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_string(provided_ioc) if provided_ioc == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855": splunk_table.append({"no data": provided_ioc}) continue if provided_ioc.lower() in usfs.extensions.keys(): ioc_dicts = query_urlscan_file(session, provided_ioc) elif validators.domain(provided_ioc) or validators.ipv4(provided_ioc) or \ validators.sha256(provided_ioc) or "certstream-suspicious" in provided_ioc: ioc_dicts = query_urlscan(session, provided_ioc) else: splunk_table.append({"invalid": provided_ioc}) continue for ioc_dict in ioc_dicts: if "ip" not in ioc_dict: splunk_table.append({"no data": provided_ioc}) continue splunk_table.append(ioc_dict) session.close() return splunk_table
def process_iocs(results): """Return data formatted for Splunk from Cymon.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() splunk_table = [] for provided_ioc in provided_iocs: provided_ioc = provided_ioc.replace("[.]", ".") provided_ioc = provided_ioc.replace("[d]", ".") provided_ioc = provided_ioc.replace("[D]", ".") if validators.ipv4(provided_ioc): ioc_type = "ip" elif validators.domain(provided_ioc): ioc_type = "domain" elif validators.md5(provided_ioc): ioc_type = "md5" elif validators.sha256(provided_ioc): ioc_type = "sha256" else: splunk_table.append({"invalid": provided_ioc}) continue ioc_dicts = query_cymon(ioc_type, session, provided_ioc) if isinstance(ioc_dicts, dict): splunk_table.append(ioc_dicts) continue for ioc_dict in ioc_dicts: ioc_dict = commons.lower_keys(ioc_dict) splunk_table.append(ioc_dict) session.close() return splunk_table
def process_iocs(results): """Return data formatted for Splunk from ThreatCrowd.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = provided_ioc.replace("[.]", ".") provided_ioc = provided_ioc.replace("[d]", ".") provided_ioc = provided_ioc.replace("[D]", ".") if validators.ipv4(provided_ioc): ioc_type = "ip" elif validators.domain(provided_ioc): ioc_type = "domain" elif validators.email(provided_ioc): ioc_type = "email" if validators.ipv4(provided_ioc) or validators.domain(provided_ioc): ioc_dicts = query_threatcrowd(provided_ioc, ioc_type, session) elif validators.email(provided_ioc): ioc_dicts = query_threatcrowd(provided_ioc, ioc_type, session) else: splunk_table.append({"invalid": provided_ioc}) continue for ioc_dict in ioc_dicts: splunk_table.append(ioc_dict) if len(provided_iocs) > 1: sleep(10) session.close() return splunk_table
def process_iocs(results): """Return data formatted for Splunk from Hybrid-Analysis.""" params = [ 'authentihash', 'av_detect', 'context', 'country', 'domain', 'env_id', 'filename', 'filetype_desc', 'filetype', 'hash', 'host', 'imp_hash', 'port', 'similar_to', 'ssdeep', 'tag', 'url', 'verdict', 'vx_family' ] if results != None: provided_iocs = [y for x in results for y in x.values()] elif sys.argv[1] == "terms" and sys.argv[2] in params: if len(sys.argv) > 2: endpoint = sys.argv[1] param = sys.argv[2] provided_iocs = sys.argv[3:] elif sys.argv[1] == "hash" and sys.argv[2] == "hash": if len(sys.argv) > 2: endpoint = sys.argv[1] param = sys.argv[2] provided_iocs = sys.argv[3:] session = commons.create_session() api_domain = commons.get_apidomain("hybrid-analysis") api_key = commons.get_apikey("hybrid-analysis") splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_string(provided_ioc) provided_ioc = provided_ioc.lower() ioc_dicts = query_hybridanalysis(endpoint, param, provided_ioc, api_domain, api_key, session) for ioc_dict in ioc_dicts: splunk_table.append(ioc_dict) session.close() return splunk_table
def get_feed(): """Return the latest report summaries from the feed.""" api = "https://ransomwaretracker.abuse.ch/feeds/csv/" session = commons.create_session() resp = session.get(api, timeout=180) if resp.status_code == 200 and resp.text != "": data = resp.text.splitlines() data = data[8:-1] data[0] = data[0][2:] header = data[0].split(",") data_feed = [] for line in data[1:]: line = line.replace('","', "^^") line = line.replace(",", " ") line = line.replace("^^", ",") line = line.replace('"', " ") ransomware_data = line.split(",") ransomware_dict = OrderedDict(zip(header, ransomware_data)) data_feed.append(ransomware_dict) return data_feed return
def process_iocs(results): """Return data formatted for Splunk from ThreatCrowd.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_string(provided_ioc) provided_ioc = provided_ioc.lower() if validators.ipv4(provided_ioc): ioc_type = "ip" elif validators.domain(provided_ioc): ioc_type = "domain" elif validators.email(provided_ioc): ioc_type = "email" elif re.match("^[a-f\d]{32}$", provided_ioc) or re.match( "^[a-f\d]{64}$", provided_ioc): ioc_type = "resource" else: splunk_table.append({"invalid": provided_ioc}) continue ioc_dicts = query_threatcrowd(provided_ioc, ioc_type, session) for ioc_dict in ioc_dicts: splunk_table.append(ioc_dict) if len(provided_iocs) > 1: sleep(10) session.close() return splunk_table
def get_feed(): """Return the latest report summaries from the feed.""" api = "https://urlhaus.abuse.ch/downloads" session = commons.create_session() resp = session.get("{}/csv/".format(api)) session.close() if resp.status_code == 200 and resp.text != "": data = resp.text.splitlines() data = data[8:] data[0] = data[0][2:] header = data[0].split(",") data_feed = [] for line in data[1:]: line = line.replace('","', "^^") line = line.replace(",", " ") line = line.replace("^^", ",") line = line.replace('"', "") ransomware_data = line.split(",") ransomware_dict = OrderedDict(zip(header, ransomware_data)) data_feed.append(ransomware_dict) return data_feed return
def process_iocs(results): """Return data formatted for Splunk from URLhaus.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() lookup_path = "/opt/splunk/etc/apps/osweep/lookups" open_file = open("{}/urlhaus_url_feed.csv".format(lookup_path), "r") contents = open_file.read().splitlines() open_file.close() header = contents[0].split(",") global data_feed data_feed = [] for line in contents: line = line.split(",") ioc_dict = OrderedDict(zip(header, line)) data_feed.append(ioc_dict) global parser parser = ParserHTML() empty_files = [ "d41d8cd98f00b204e9800998ecf8427e", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ] urlhaus_match = re.compile(r"^h..ps?:\/\/urlhaus\.abuse\.ch") splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_url(provided_ioc) if provided_ioc in empty_files: splunk_table.append({"invalid": provided_ioc}) continue if urlhaus_match.match(provided_ioc): splunk_table.append({"invalid": provided_ioc}) continue if validators.url(provided_ioc) or validators.domain(provided_ioc) or \ validators.ipv4(provided_ioc): analysis_dicts = get_analysis(provided_ioc) if isinstance(analysis_dicts, dict): splunk_table.append(analysis_dicts) continue ioc_dicts = get_payloads(analysis_dicts, session) elif validators.md5(provided_ioc) or validators.sha256(provided_ioc): ioc_dicts = get_urls(session, provided_ioc) else: splunk_table.append({"invalid": provided_ioc}) continue for ioc_dict in ioc_dicts: ioc_dict = commons.lower_keys(ioc_dict) splunk_table.append(ioc_dict) time.sleep(1) session.close() return splunk_table