def __init__(self, ax_report: str):
data = json.load(open(ax_report, "r"))
self.version: str = data.get("version", "8.1.0")
self.appliance = data.get("appliance", "Unknown")
if "alert" not in data or len(data["alert"]) == 0:
self.alert = {} # type: ignore
else:
self.alert = data["alert"][0]
occured = self.alert["occurred"].replace(" +0000", "")
# Multiple possible timestamp string.
for fmt_string in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"]:
try:
self.base_timestamp = int(
datetime.datetime.strptime(occured, fmt_string).timestamp()
)
break # break if succesfully parsed.
except ValueError:
continue
# If break not reached.
else:
raise ValueError(
f"{self.alert['occurred']} did not match any known time format strings for AX"
)
logger.info("Set up FireEyeAX Report")
def __init__(self, procmon_csv: str) -> None:
self._df = pd.read_csv(procmon_csv)
# Procmon doesn't have dates time.
self.now = datetime.datetime.now()
logger.info("Set up ProcmonCSVs")
def __init__(self, file_hash: str, sandbox_name: str = None):
api_key = Config.get("virustotal", "api_key")
if not api_key:
logger.critical(
f"BEAGLE__VIRUSTOTAL__API_KEY not found in enviroment variables or beagle.config object"
)
raise RuntimeError(
"BEAGLE__VIRUSTOTAL__API_KEY not found in enviroment variables or beagle.config object"
)
logger.info(f"Grabbing metadata and sandbox reports for {file_hash}")
headers = {"x-apikey": api_key}
self.hash_metadata = requests.get(
f"https://www.virustotal.com/api/v3/files/{file_hash}", headers=headers
).json()
behaviour_reports = requests.get(
f"https://www.virustotal.com/api/v3/files/{file_hash}/behaviours", headers=headers
).json()
# Get the sandbox we want, or the first one.
if sandbox_name:
possible_sandboxes = [
report["attributes"]["sandbox_name"] for report in behaviour_reports["data"]
]
logger.info(f"Sample has reports from {','.join(possible_sandboxes)}")
if sandbox_name in possible_sandboxes:
logger.info(f"Requested sandbox {sandbox_name} availble, using it.")
behaviour_report = list(
filter(
lambda val: val["attributes"]["sandbox_name"] == sandbox_name,
behaviour_reports["data"],
)
)[0]
else:
logger.info(f"Requested sandbox {sandbox_name} not found, using first sandbox.")
behaviour_report = behaviour_reports["data"][0]
else:
behaviour_report = behaviour_reports["data"][0]
logger.info(
f"No sandbox specified, using {behaviour_report['attributes']['sandbox_name']}"
)
self.behaviour_report = behaviour_report[
"attributes"
] # Set up same way as GenericVTSandbox
def setup_session(self): # pragma: no cover
import splunklib.client as client
client_kwargs = {
"host": Config.get("splunk", "host"),
"username": Config.get("splunk", "username"),
"password": Config.get("splunk", "password"),
"port": int(Config.get("splunk", "port", fallback=8089)),
}
logger.info(f"Creating Splunk client for host={client_kwargs['host']}")
return client.connect(sharing="global",
**client_kwargs,
handler=handler())
def __init__(self, ax_report: str):
data = json.load(open(ax_report, "r"))
self.appliance = data.get("appliance", "Unknown")
if "alert" not in data or len(data["alert"]) == 0:
self.alert = {} # type: ignore
else:
self.alert = data["alert"][0]
self.base_timestamp = int(
datetime.datetime.strptime(
self.alert["occurred"].replace(" +0000", ""),
"%Y-%m-%d %H:%M:%S").timestamp())
logger.info("Set up FireEyeAX Report")
def _setup_session(self): # pragma: no cover
from elasticsearch import Elasticsearch
client_kwargs = {
"host": Config.get("elasticsearch", "host"),
"scheme": Config.get("elasticsearch", "scheme"),
"port": int(Config.get("elasticsearch", "port", fallback=9200)),
}
if Config.get("elasticsearch", "username") and Config.get("elasticsearch", "password"):
client_kwargs["http_auth"] = (
Config.get("elasticsearch", "username"),
Config.get("elasticsearch", "password"),
)
logger.info(f"Creating Elasticsearch client for host={client_kwargs['host']}")
return Elasticsearch(**client_kwargs)
def events(self) -> Generator[dict, None, None]:
from splunklib.client import Job
job: Job = self.create_search(
self.spl,
query_kwargs={
"exec_mode": "normal",
"earliest_time": self.earliest,
"latest_time": self.latest,
},
)
self.sid = job.sid
logger.info(
f"Creating splunk search with sid={self.sid}, waiting for job=Done"
)
while not job.is_done():
logger.debug("Job not done, sleeping")
time.sleep(5)
logger.info(f"Job is done, getting results")
count = 0
for result in self.get_results(job, count=100000000):
count += 1
yield result
logger.info(f"Processed {count} splunk results")
def __init__(self,
behaviour_report_file: str,
hash_metadata_file: str = None) -> None:
behaviour_report = json.load(open(behaviour_report_file, "r"))
if "attributes" not in behaviour_report:
raise AttributeError(
f"Supplied behaviour report does not contain any data")
hash_metadata = None
if hash_metadata_file:
hash_metadata = json.load(open(hash_metadata_file, "r"))
self.hash_metadata = hash_metadata
self.behaviour_report = behaviour_report["attributes"]
logger.info(f"Finished setting up GenericVTSandbox")
for key in self.behaviour_report.keys():
if key not in self.KNOWN_ATTRIBUTES:
logger.debug(
f"Unknown key {key} found in VirusTotal sandbox report")
def __init__(self, triage: str) -> None:
"""A FireEye HX Triage DataSource.
Parameters
----------
triage : str
The path to the HX .mans file.
Examples
-------
>>> triage = HXTriage(triage="/path/to/triage.mans")
"""
self.file_path = triage
self.alert_files = {"hits.json": False, "threats.json": False}
self.identified_files: Dict[str, str] = {}
logger.info(f"Setting up HXTriage for {self.file_path}")
self.tempdir = tempfile.TemporaryDirectory(suffix="_beagle")
logger.debug(f"Generated temporary directory {self.tempdir.name}")
# .mans files are simply zips.
with zipfile.ZipFile(self.file_path) as mans:
for audit_file in mans.namelist():
# Save the alert files
if audit_file in self._ALERT_FILES:
with mans.open(audit_file) as f:
mans.extract(audit_file, f"{self.tempdir.name}")
self.alert_files[audit_file] = True
logger.debug(f"Found alert file {audit_file}")
# Skip files with '.' in them
if "." in audit_file:
continue
# Get the audit type.
with mans.open(audit_file) as f:
header = f.read(500).decode("utf-8")
match = self._GENERATOR_REX.search(header)
if match:
version = match.groups()[0]
else:
continue
# Skip if not supported
if version not in self._SUPPORTED_AUDITS:
continue
mans.extract(audit_file, f"{self.tempdir.name}")
self.identified_files[audit_file] = version
logger.debug(f"Mapped {audit_file} to {version}")
def parse_alert_files(self, temp_dir: str) -> Generator[dict, None, None]:
"""Parses out the alert files from the hits.json and threats.json files
Parameters
----------
temp_dir : str
Folder which contains the expanded triage.
Yields
-------
Generator[dict, None, None]
The next event found in the Triage.
"""
threats = None
# We will always have 'hits.json'
try:
hits = json.load(open(f"{temp_dir}/hits.json", "r"))
except Exception as e:
logger.warning(f"Could not load JSON from hits.json, skipping alerts!")
logger.debug(e)
return
if self.alert_files["threats.json"]:
try:
threats = json.load(open(f"{temp_dir}/threats.json", "r"))
except Exception as e:
logger.warning(f"Could not load JSON from threats.json, alert names may be UUIDs")
logger.debug(e)
else:
logger.info(f"Could not find threats.json, alert names may be UUIDs")
for alert in hits:
# Introduce an alerting event type for HX.
alert["event_type"] = "alertEvent"
# If we have the threats file, convert the "threat_id" portion of the alert to
# understandable values.
if threats:
threat = next(
filter(lambda threat_entry: threat_entry["_id"] == alert["threat_id"], threats),
None,
)
if threat:
logger.info(
f"Matched up {alert['threat_id']} to {threat.get('display_name', threat['uri_name'])}"
)
alert["_threat_data"] = threat
# Add the time the alert happend
alert["event_time"] = self._hx_time_to_epoch(alert["matched_at"])
alerting_event_type = alert["data"]["key"]["event_type"]
# Strip the event type
alerting_event = {
k.replace(f"{alerting_event_type}/", ""): v
for k, v in alert["data"]["values"].items()
}
alerting_event["event_time"] = self._hx_time_to_epoch(alerting_event["timestamp"])
alert["data"]["values"] = self._fix_missing_fields(alerting_event)
yield alert
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
logger.info("Created Generic Transformer.")
def __init__(self, evtx_log_file: str) -> None:
self.file_path = evtx_log_file
logger.info(f"Setting up WinEVTX for {self.file_path}")
def __init__(self, cuckoo_report: str) -> None:
self.report = json.load(open(cuckoo_report, "r"))
self.behavior = self.report["behavior"]
self.processes: Dict[int, dict] = {}
logger.info("Set up Cuckoo Sandbox")
