async def _run(args: Tuple[int, str, dict]) -> Union[Dict[str, Any], None]:
"""Handle our multiprocessing tasks."""
count, path, indicators = args
_indicators = [(indicator["name"], indicator["indicator"]["rule"])
for indicator in indicators]
yara_rules = compile_rules(tuple(_indicators))
ignorelist = [
"\\OneDrive\\",
"\\OneDriveTemp\\",
os.getcwd(),
] # Ignore these paths, so we don't enumerate cloud drives or our current working directory
if count % 50000 == 0 and count != 0:
CONSOLE(
"[cyan][YARA][/cyan] We're still working on scanning files. {} processed."
.format(count))
if count == 1:
CONSOLE("[cyan][YARA][/cyan] Beginning processing.")
# Sometimes glob.glob gives us paths with *
if (os.path.exists(path)
and (path != "." and path != "\\" and all(x not in path
for x in ignorelist))
and not os.path.isdir(path)):
try:
matches = yara_rules.match(path)
if matches:
for match in matches:
attrs = [
"meta", "namespace", "rule", "strings", "tags"
]
match_dict = {k: str(getattr(match, k)) for k in attrs}
match_dict["file"] = path
return match_dict
except yara.Error:
pass
async def run(indicators: dict) -> None:
"""Accept a dict containing events indicators and write out to the OUTPUT_DIR specified by chirp.common.
:param indicators: A dict containing parsed registry indicator files.
:type indicators: dict
"""
if not indicators:
return
CONSOLE("[cyan][REGISTRY][/cyan] Entered registry plugin.")
report = {
indicator["name"]: build_report(indicator)
for indicator in indicators
}
for indicator in indicators:
ind = indicator["indicator"]
indicator_list = [(k, v) for k, v in ind.items()
if k != "registry_key"]
CONSOLE("[cyan][REGISTRY][/cyan] Reading {}".format(
ind["registry_key"]))
async for value in enumerate_registry_values(ind["registry_key"]):
if value == "ERROR":
CONSOLE("[cyan][REGISTRY][/cyan] Hit an error, exiting.")
return
hits, search_criteria, match = await check_matches(
indicator_list, value)
if hits != len(indicator_list):
continue
report[indicator["name"]]["_search_criteria"] = search_criteria
if match:
report[indicator["name"]]["matches"].append(match)
[await _report_hits(k, v) for k, v in report.items()]
with open(os.path.join(OUTPUT_DIR, "registry.json"), "w+") as writeout:
writeout.write(
json.dumps({r: report[r]
for r in report if report[r]["matches"]}))
async def run(indicators: dict) -> None:
"""Accept a dict containing events indicators and writes out to the OUTPUT_DIR specified by chirp.common.
:param indicators: A dict containing parsed network indicator files.
:type indicators: dict
"""
if not indicators:
return
hits = 0
CONSOLE("[cyan][NETWORK][/cyan] Entered network plugin.")
saved_ns = parse_netstat(grab_netstat())
saved_dns = parse_dns(grab_dns())
report = {
indicator["name"]: build_report(indicator)
for indicator in indicators
}
for indicator in indicators:
try:
for ioc in indicator["indicator"]["ips"].splitlines():
if await hunter(saved_ns + ["\n"] + saved_dns, ioc):
report[indicator["name"]]["matches"].append(ioc)
hits += 1
except KeyError:
ERROR("{} appears to be malformed.".format(indicator))
CONSOLE(
"[cyan][NETWORK][/cyan] Read {} records, found {} IoC hits.".format(
len(saved_dns) + len(saved_ns), hits))
with open(os.path.join(OUTPUT_DIR, "network.json"), "w+") as writeout:
writeout.write(
json.dumps({r: report[r]
for r in report if report[r]["matches"]}))
async def enumerate_registry_values(hkey: str) -> Iterator[dict]:
"""Enumerate the values of the given key.
:param hkey: A registry key to enumerate
:type hkey: str
:yield: Registry key values
:rtype: Iterator[dict]
"""
hive, key = _normalize_key(hkey)
if not hive or not key:
CONSOLE("[red][!][/red] Unable to read key '{}'".format(hkey))
return
registry = winreg.ConnectRegistry(None, hive)
try:
with winreg.OpenKey(registry, key) as registry_key:
for i in range(winreg.QueryInfoKey(registry_key)[1]):
value_tuple = winreg.EnumValue(registry_key, i)
yield {
"key": value_tuple[0],
"value": value_tuple[1],
"registry_type": REGISTRY_VALUE_TYPES[value_tuple[2]],
}
except FileNotFoundError:
CONSOLE(
"[cyan][REGISTRY][/cyan] Key {} does not exist.".format(hkey))
async def run(indicators: dict) -> None:
"""Accept a dict containing yara indicators and write out to the OUTPUT_DIR specified by chirp.common.
:param indicators: A NamespaceDict containing parsed yara indicator files.
:type indicators: dict
"""
if not indicators:
return
CONSOLE("[cyan][YARA][/cyan] Entered yara plugin.")
files = [i["indicator"]["files"] for i in indicators]
files = "\\**" if "\\**" in files else ", ".join(files)
if files == "\\**":
blame = [
i["name"] for i in indicators
if i["indicator"]["files"] == "\\**"
]
CONSOLE(
"[cyan][YARA][/cyan] Enumerating the entire filesystem due to {}... this is going to take a while."
.format(blame))
report = {
indicator["name"]: build_report(indicator)
for indicator in indicators
}
hits = 0
run_args = []
# Normalize every path, for every path
try:
run_args = [(a, b, indicators)
for a, b in enumerate(normalize_paths(files), 1)]
async with aiomp.Pool() as pool:
try:
async for result in pool.map(_run, tuple(run_args)):
if result:
report[result["namespace"]]["matches"].append(
result)
hits += 1
except KeyboardInterrupt:
pass
except IndexError:
pass
count = len(run_args)
CONSOLE("[cyan][YARA][/cyan] Done. Processed {} files.".format(count))
CONSOLE(
"[cyan][YARA][/cyan] Found {} hit(s) for yara indicators.".format(
hits))
with open(os.path.join(OUTPUT_DIR, "yara.json"), "w+") as writeout:
writeout.write(
json.dumps(
{r: report[r]
for r in report if report[r]["matches"]}))
def iter_evtx2xml(evtx_file):
"""
Generator function to read events from evtx file and convert to xml
:param evtx_file: file path string
:return: generator to xml string representation of evtx event
"""
global error_counter, event_counter
error_counter = 0
event_counter = 0
try:
with evtx.Evtx(evtx_file) as log:
# process each log entry and return xml representation
for record in log.records():
event_counter += 1
try:
yield record.xml()
except Exception as err:
error_counter += 1
# logger.error("Failed to convert EVTX to XML for %s. Error count: %d" % (evtx_file, error_counter))
except Exception as err:
raise
if error_counter:
CONSOLE(
"[cyan][evtx2json][/cyan] Failed to read {} events.".format(error_counter)
)
async def enumerate_registry_values(hkey: str) -> Iterator[str]:
"""Return if the proper libraries can't be imported (like wrong OS).
:param hkey: A registry key to query
:type hkey: str
:yield: Literally "ERROR"
:rtype: Iterator[str]
"""
CONSOLE("[red][!][/red] Registry plugin is only compatible with Windows.")
yield "ERROR"
async def _run(run_args):
"""Gather events and check for matches."""
(
event_type,
indicators,
report,
num_logs,
) = run_args # Unpack our arguments (bundled to passthrough for multiprocessing)
CONSOLE("[cyan][EVENTS][/cyan] Reading {} event logs.".format(
event_type.split("%4")[-1]))
async for event_log in gather(event_type): # Iterate over event logs
if event_log == "ERROR":
CONSOLE("[cyan][EVENTS][/cyan] Hit an error, exiting.")
return
if event_log:
num_logs += 1
for indicator in indicators:
ind = indicator["indicator"]
if (ind["event_type"] == event_type
): # Make sure the ioc is intended for this log
if "event_id" in ind and str(
event_log["event"]["system"]["event_id"]["$"]
) != str(
ind["event_id"]
): # If ioc looks for event_id, but there is a mismatch then skip
continue
elif "event_id" not in ind:
ind["event_id"] = None
indicator_list = [(k, v) for k, v in ind.items()
if k not in ["event_type", "event_id"]]
hits, search_criteria, match = await check_matches(
indicator_list, ind["event_id"], event_log
) # Check to see if the indicator matches the event log
if hits != len(indicator_list):
continue
report[indicator["name"]][
"_search_criteria"] = search_criteria
if match:
report[indicator["name"]]["matches"].append(
match
) # Append to report because there is a match.
return report, num_logs
async def run(indicators: dict) -> None:
"""Accept a dict containing events indicators and writes out to the OUTPUT_DIR specified by chirp.common.
:param indicators: A dict containing parsed events indicator files.
:type indicators: dict
"""
if not indicators:
return
hits = 0
num_logs = 0
CONSOLE("[cyan][EVENTS][/cyan] Entered events plugin.")
event_types = {
indicator["indicator"]["event_type"]
for indicator in indicators
}
report = {
indicator["name"]: build_report(indicator)
for indicator in indicators
}
run_args = [(event_type, indicators, report, num_logs)
for event_type in event_types]
async with aiomp.Pool() as pool:
try:
async for i in pool.map(_run, tuple(run_args)):
_rep = i[0]
num_logs += i[1]
for k, v in _rep.items():
try:
report[k]["_search_criteria"] = v["_search_criteria"]
except KeyError:
pass
report[k]["matches"] += v["matches"]
except KeyboardInterrupt:
pass
hits = sum(len(v["matches"]) for _, v in report.items())
CONSOLE("[cyan][EVENTS][/cyan] Read {} logs, found {} matches.".format(
num_logs, hits))
with open(os.path.join(OUTPUT_DIR, "events.json"), "w+") as writeout:
writeout.write(
json.dumps({r: report[r]
for r in report if report[r]["matches"]}))
"""Main method for CHIRP (Used when compiled)."""
# Standard Python Libraries
from multiprocessing import freeze_support
import os
# cisagov Libraries
from chirp.common import CONSOLE, ERROR, OUTPUT_DIR, save_log
import chirp.run
if __name__ == "__main__":
try:
freeze_support()
chirp.run.run()
CONSOLE(
"[green][+][/green] DONE! Your results can be found in {}.".format(
os.path.abspath(OUTPUT_DIR)
)
)
except KeyboardInterrupt:
ERROR("Received an escape sequence. Goodbye.")
finally:
save_log()
input()
pass
except IndexError:
pass
count = len(run_args)
CONSOLE("[cyan][YARA][/cyan] Done. Processed {} files.".format(count))
CONSOLE(
"[cyan][YARA][/cyan] Found {} hit(s) for yara indicators.".format(
hits))
with open(os.path.join(OUTPUT_DIR, "yara.json"), "w+") as writeout:
writeout.write(
json.dumps(
{r: report[r]
for r in report if report[r]["matches"]}))
else:
CONSOLE(
"[red][!][/red] yara-python is a required dependency for the yara plugin. Please install yara-python with pip."
)
CONSOLE("[cyan][YARA][/cyan] Hit an error, exiting.")
async def run(indicators: dict) -> None:
"""Return if there is an import error.
:param indicators: Parsed yara indicator files.
:type indicators: dict
"""
return
import string
import sys
from typing import Any, Dict, Iterator, List, Union
# cisagov Libraries
from chirp.common import CONSOLE, JSON, OS
HAS_LIBS = False
try:
# cisagov Libraries
from chirp.plugins.events.evtx2json import iter_evtx2xml, splunkify, xml2json
HAS_LIBS = True
except ImportError:
CONSOLE(
"[red][!][/red] python-evtx, dict-toolbox, and xmljson are required dependencies for the events plugin. Please install requirements with pip."
)
if OS == "Windows":
# Standard Python Libraries
from ctypes import windll
PATH = Path(sys.executable)
def _get_drives() -> List[str]:
"""
Return a list of valid drives.
Reference: `RichieHindle, StackOverflow <https://stackoverflow.com/a/827398>`_
"""
async def _report_hits(indicator: str, vals: dict) -> None:
"""Write to the log the number of hits for a given indicator."""
CONSOLE("[cyan][REGISTRY][/cyan] Found {} hit(s) for {} indicator.".format(
len(vals["matches"]), indicator))
import sys
from typing import Any, Dict, Iterator, List, Union
# cisagov Libraries
from chirp.common import CONSOLE, JSON
HAS_LIBS = False
try:
# cisagov Libraries
from chirp.common import OS
from chirp.plugins.events.evtx2json import iter_evtx2xml, splunkify, xml2json
HAS_LIBS = True
except ImportError:
CONSOLE(
"[red][!][/red] python-evtx, dict-toolbox, and xmljson are required dependencies for the events plugin. Please install requirements with pip."
)
PATH = Path(sys.executable)
# Depending on if this is 32bit or 64bit Windows, the logs can be at System32 or Sysnative.
if os.path.exists(PATH.drive + "\\Windows\\Sysnative\\winevt"):
default_dir = PATH.drive + "\\Windows\\Sysnative\\winevt\\Logs\\{}.evtx"
elif os.path.exists(PATH.drive + "\\Windows\\System32\\winevt"):
default_dir = PATH.drive + "\\Windows\\System32\\winevt\\Logs\\{}.evtx"
else:
# TODO: Implement switch
if OS == "Windows":
CONSOLE(
"[red][!][/red] We can't find the windows event logs at {}\\System32\\winevt or at {}\\Sysnative\\winevt."
.format(PATH.drive, PATH.drive))