def query_threatcrowd(provided_ioc, ioc_type, session): """Pivot off an IP or domain and return data as an dictonary.""" ioc_dicts = [] if ioc_type == "resource": resp = session.get(api.format("file", ioc_type, provided_ioc), timeout=180) else: resp = session.get(api.format(ioc_type, ioc_type, provided_ioc), timeout=180) if resp.status_code == 200 and "permalink" in resp.json().keys() and \ provided_ioc in resp.json()["permalink"]: for key in resp.json().keys(): if key == "votes" or key == "permalink" or key == "response_code": continue elif key in ("md5", "sha1"): value = resp.json()[key] ioc_dicts.append({key: value}) elif key == "resolutions": for res in resp.json()[key]: res = commons.lower_keys(res) ioc_dicts.append(res) else: for value in resp.json()[key]: key = commons.lower_keys(key) ioc_dicts.append({key: value}) else: ioc_dicts.append({"no data": provided_ioc}) return ioc_dicts
def process_iocs(results): """Return data formatted for Splunk from Twitter.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] if len(provided_iocs) > 180: return { "error": "Search term limit: 180\nTotal Search Terms Provided: {}".format( len(provided_iocs)) } session = create_session() splunk_table = [] if isinstance(session, dict): splunk_table.append(session) return splunk_table rate_limit = check_rate_limit(session, provided_iocs) if isinstance(rate_limit, dict): splunk_table.append(rate_limit) return splunk_table empty_files = [ "d41d8cd98f00b204e9800998ecf8427e", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ] splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = provided_ioc.replace("[.]", ".") provided_ioc = provided_ioc.replace("[d]", ".") provided_ioc = provided_ioc.replace("[D]", ".") if provided_ioc in empty_files: splunk_table.append({"invalid": provided_ioc}) continue if validators.url(provided_ioc) or validators.domain(provided_ioc) or \ validators.ipv4(provided_ioc) or validators.md5(provided_ioc) or \ validators.sha256(provided_ioc) or \ len(provided_ioc) > 2 and len(provided_ioc) <= 140: ioc_dicts = query_twitter(session, provided_ioc) else: splunk_table.append({"invalid": provided_ioc}) continue for ioc_dict in ioc_dicts: ioc_dict = commons.lower_keys(ioc_dict) splunk_table.append(ioc_dict) return splunk_table
def process_iocs(results): """Return data formatted for Splunk from URLhaus.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() empty_files = [ "d41d8cd98f00b204e9800998ecf8427e", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ] urlhaus_match = re.compile(r"^h..ps?:\/\/urlhaus\.abuse\.ch") splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_string(provided_ioc) if provided_ioc in empty_files: splunk_table.append({"invalid": provided_ioc}) continue if urlhaus_match.match(provided_ioc): splunk_table.append({"invalid": provided_ioc}) continue if validators.domain(provided_ioc) or validators.ipv4(provided_ioc): ioc_type = "host" elif validators.url(provided_ioc): ioc_type = "url" elif re.match("^[a-f\d]{32}$", provided_ioc) or re.match( "^[a-f\d]{64}$", provided_ioc): ioc_type = "payload" else: splunk_table.append({"invalid": provided_ioc}) continue ioc_dicts = query_urlhaus(session, provided_ioc, ioc_type) for ioc_dict in ioc_dicts: ioc_dict = commons.lower_keys(ioc_dict) splunk_table.append(ioc_dict) session.close() return splunk_table
def psbdmp_search(provided_ioc, session): """ """ base_url = "https://psbdmp.ws/api/search/{}" url = base_url.format(provided_ioc) resp = session.get(url, timeout=180) psd_dicts = [] if resp.status_code == 200 and resp.json()["error"] != 1 and len( resp.json()["data"]) > 0: data = resp.json()["data"] for result in data: result = commons.lower_keys(result) result.update({"provided_ioc": provided_ioc}) psd_dicts.append(result) else: psd_dicts.append({"no data": provided_ioc}) return psd_dicts
def process_iocs(results): """Return data formatted for Splunk from GreyNoise.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] splunk_table = [] lookup_path = "/opt/splunk/etc/apps/osweep/lookups" open_file = open("{}/greynoise_feed.csv".format(lookup_path), "r") data_feed = open_file.read().splitlines() header = data_feed[0].split(",") open_file.close() open_file = open("{}/greynoise_scanners.csv".format(lookup_path), "r") scanners = set(open_file.read().splitlines()[1:]) scanners = [x.lower() for x in scanners] open_file.close() for provided_ioc in set(provided_iocs): provided_ioc = provided_ioc.replace("[.]", ".") provided_ioc = provided_ioc.replace("[d]", ".") provided_ioc = provided_ioc.replace("[D]", ".") if not validators.ipv4(provided_ioc) and \ not validators.domain(provided_ioc) and \ provided_ioc.lower() not in scanners: splunk_table.append({"invalid": provided_ioc}) continue line_found = False for line in data_feed: if provided_ioc.lower() in line.lower(): line_found = True scanner_data = line.split(",") scanner_dict = OrderedDict(zip(header, scanner_data)) scanner_dict = commons.lower_keys(scanner_dict) splunk_table.append(scanner_dict) if line_found == False: splunk_table.append({"no data": provided_ioc}) return splunk_table
def process_iocs(results): """Return data formatted for Splunk from Cymon.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() splunk_table = [] for provided_ioc in provided_iocs: provided_ioc = provided_ioc.replace("[.]", ".") provided_ioc = provided_ioc.replace("[d]", ".") provided_ioc = provided_ioc.replace("[D]", ".") if validators.ipv4(provided_ioc): ioc_type = "ip" elif validators.domain(provided_ioc): ioc_type = "domain" elif validators.md5(provided_ioc): ioc_type = "md5" elif validators.sha256(provided_ioc): ioc_type = "sha256" else: splunk_table.append({"invalid": provided_ioc}) continue ioc_dicts = query_cymon(ioc_type, session, provided_ioc) if isinstance(ioc_dicts, dict): splunk_table.append(ioc_dicts) continue for ioc_dict in ioc_dicts: ioc_dict = commons.lower_keys(ioc_dict) splunk_table.append(ioc_dict) session.close() return splunk_table
def query_crtsh(provided_ioc, session): """Search crt.sh for the given domain.""" if sys.argv[1] == "subdomain": provided_ioc = "%25.{}".format(provided_ioc) elif sys.argv[1] == "wildcard": provided_ioc = "%25{}".format(provided_ioc) base_url = "https://crt.sh/?q={}&output=json" url = base_url.format(provided_ioc) resp = session.get(url, timeout=180) crt_dicts = [] if resp.status_code == 200 and resp.content != "": content = resp.content.decode("UTF-8") cert_history = json.loads("[{}]".format(content.replace("}{", "},{"))) for cert in cert_history: cert = commons.lower_keys(cert) crt_dicts.append(cert) else: provided_ioc = provided_ioc.replace("%25.", "").replace("%25", "") crt_dicts.append({"no data": provided_ioc}) return crt_dicts
def process_iocs(results): """Return data formatted for Splunk from URLhaus.""" if results != None: provided_iocs = [y for x in results for y in x.values()] else: provided_iocs = sys.argv[1:] session = commons.create_session() lookup_path = "/opt/splunk/etc/apps/osweep/lookups" open_file = open("{}/urlhaus_url_feed.csv".format(lookup_path), "r") contents = open_file.read().splitlines() open_file.close() header = contents[0].split(",") global data_feed data_feed = [] for line in contents: line = line.split(",") ioc_dict = OrderedDict(zip(header, line)) data_feed.append(ioc_dict) global parser parser = ParserHTML() empty_files = [ "d41d8cd98f00b204e9800998ecf8427e", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ] urlhaus_match = re.compile(r"^h..ps?:\/\/urlhaus\.abuse\.ch") splunk_table = [] for provided_ioc in set(provided_iocs): provided_ioc = commons.deobfuscate_url(provided_ioc) if provided_ioc in empty_files: splunk_table.append({"invalid": provided_ioc}) continue if urlhaus_match.match(provided_ioc): splunk_table.append({"invalid": provided_ioc}) continue if validators.url(provided_ioc) or validators.domain(provided_ioc) or \ validators.ipv4(provided_ioc): analysis_dicts = get_analysis(provided_ioc) if isinstance(analysis_dicts, dict): splunk_table.append(analysis_dicts) continue ioc_dicts = get_payloads(analysis_dicts, session) elif validators.md5(provided_ioc) or validators.sha256(provided_ioc): ioc_dicts = get_urls(session, provided_ioc) else: splunk_table.append({"invalid": provided_ioc}) continue for ioc_dict in ioc_dicts: ioc_dict = commons.lower_keys(ioc_dict) splunk_table.append(ioc_dict) time.sleep(1) session.close() return splunk_table
def merge_dict(page, download): """Return a dictionary containing both page and download data.""" merged_dict = {} merged_dict.update(commons.lower_keys(page)) merged_dict.update(commons.lower_keys(download)) return merged_dict