def __init__(self, ax_report: str): data = json.load(open(ax_report, "r")) self.version: str = data.get("version", "8.1.0") self.appliance = data.get("appliance", "Unknown") if "alert" not in data or len(data["alert"]) == 0: self.alert = {} # type: ignore else: self.alert = data["alert"][0] occured = self.alert["occurred"].replace(" +0000", "") # Multiple possible timestamp string. for fmt_string in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"]: try: self.base_timestamp = int( datetime.datetime.strptime(occured, fmt_string).timestamp() ) break # break if succesfully parsed. except ValueError: continue # If break not reached. else: raise ValueError( f"{self.alert['occurred']} did not match any known time format strings for AX" ) logger.info("Set up FireEyeAX Report")
def __init__(self, procmon_csv: str) -> None: self._df = pd.read_csv(procmon_csv) # Procmon doesn't have dates time. self.now = datetime.datetime.now() logger.info("Set up ProcmonCSVs")
def __init__(self, file_hash: str, sandbox_name: str = None): api_key = Config.get("virustotal", "api_key") if not api_key: logger.critical( f"BEAGLE__VIRUSTOTAL__API_KEY not found in enviroment variables or beagle.config object" ) raise RuntimeError( "BEAGLE__VIRUSTOTAL__API_KEY not found in enviroment variables or beagle.config object" ) logger.info(f"Grabbing metadata and sandbox reports for {file_hash}") headers = {"x-apikey": api_key} self.hash_metadata = requests.get( f"https://www.virustotal.com/api/v3/files/{file_hash}", headers=headers ).json() behaviour_reports = requests.get( f"https://www.virustotal.com/api/v3/files/{file_hash}/behaviours", headers=headers ).json() # Get the sandbox we want, or the first one. if sandbox_name: possible_sandboxes = [ report["attributes"]["sandbox_name"] for report in behaviour_reports["data"] ] logger.info(f"Sample has reports from {','.join(possible_sandboxes)}") if sandbox_name in possible_sandboxes: logger.info(f"Requested sandbox {sandbox_name} availble, using it.") behaviour_report = list( filter( lambda val: val["attributes"]["sandbox_name"] == sandbox_name, behaviour_reports["data"], ) )[0] else: logger.info(f"Requested sandbox {sandbox_name} not found, using first sandbox.") behaviour_report = behaviour_reports["data"][0] else: behaviour_report = behaviour_reports["data"][0] logger.info( f"No sandbox specified, using {behaviour_report['attributes']['sandbox_name']}" ) self.behaviour_report = behaviour_report[ "attributes" ] # Set up same way as GenericVTSandbox
def setup_session(self): # pragma: no cover import splunklib.client as client client_kwargs = { "host": Config.get("splunk", "host"), "username": Config.get("splunk", "username"), "password": Config.get("splunk", "password"), "port": int(Config.get("splunk", "port", fallback=8089)), } logger.info(f"Creating Splunk client for host={client_kwargs['host']}") return client.connect(sharing="global", **client_kwargs, handler=handler())
def __init__(self, ax_report: str): data = json.load(open(ax_report, "r")) self.appliance = data.get("appliance", "Unknown") if "alert" not in data or len(data["alert"]) == 0: self.alert = {} # type: ignore else: self.alert = data["alert"][0] self.base_timestamp = int( datetime.datetime.strptime( self.alert["occurred"].replace(" +0000", ""), "%Y-%m-%d %H:%M:%S").timestamp()) logger.info("Set up FireEyeAX Report")
def _setup_session(self): # pragma: no cover from elasticsearch import Elasticsearch client_kwargs = { "host": Config.get("elasticsearch", "host"), "scheme": Config.get("elasticsearch", "scheme"), "port": int(Config.get("elasticsearch", "port", fallback=9200)), } if Config.get("elasticsearch", "username") and Config.get("elasticsearch", "password"): client_kwargs["http_auth"] = ( Config.get("elasticsearch", "username"), Config.get("elasticsearch", "password"), ) logger.info(f"Creating Elasticsearch client for host={client_kwargs['host']}") return Elasticsearch(**client_kwargs)
def events(self) -> Generator[dict, None, None]: from splunklib.client import Job job: Job = self.create_search( self.spl, query_kwargs={ "exec_mode": "normal", "earliest_time": self.earliest, "latest_time": self.latest, }, ) self.sid = job.sid logger.info( f"Creating splunk search with sid={self.sid}, waiting for job=Done" ) while not job.is_done(): logger.debug("Job not done, sleeping") time.sleep(5) logger.info(f"Job is done, getting results") count = 0 for result in self.get_results(job, count=100000000): count += 1 yield result logger.info(f"Processed {count} splunk results")
def __init__(self, behaviour_report_file: str, hash_metadata_file: str = None) -> None: behaviour_report = json.load(open(behaviour_report_file, "r")) if "attributes" not in behaviour_report: raise AttributeError( f"Supplied behaviour report does not contain any data") hash_metadata = None if hash_metadata_file: hash_metadata = json.load(open(hash_metadata_file, "r")) self.hash_metadata = hash_metadata self.behaviour_report = behaviour_report["attributes"] logger.info(f"Finished setting up GenericVTSandbox") for key in self.behaviour_report.keys(): if key not in self.KNOWN_ATTRIBUTES: logger.debug( f"Unknown key {key} found in VirusTotal sandbox report")
def __init__(self, triage: str) -> None: """A FireEye HX Triage DataSource. Parameters ---------- triage : str The path to the HX .mans file. Examples ------- >>> triage = HXTriage(triage="/path/to/triage.mans") """ self.file_path = triage self.alert_files = {"hits.json": False, "threats.json": False} self.identified_files: Dict[str, str] = {} logger.info(f"Setting up HXTriage for {self.file_path}") self.tempdir = tempfile.TemporaryDirectory(suffix="_beagle") logger.debug(f"Generated temporary directory {self.tempdir.name}") # .mans files are simply zips. with zipfile.ZipFile(self.file_path) as mans: for audit_file in mans.namelist(): # Save the alert files if audit_file in self._ALERT_FILES: with mans.open(audit_file) as f: mans.extract(audit_file, f"{self.tempdir.name}") self.alert_files[audit_file] = True logger.debug(f"Found alert file {audit_file}") # Skip files with '.' in them if "." in audit_file: continue # Get the audit type. with mans.open(audit_file) as f: header = f.read(500).decode("utf-8") match = self._GENERATOR_REX.search(header) if match: version = match.groups()[0] else: continue # Skip if not supported if version not in self._SUPPORTED_AUDITS: continue mans.extract(audit_file, f"{self.tempdir.name}") self.identified_files[audit_file] = version logger.debug(f"Mapped {audit_file} to {version}")
def parse_alert_files(self, temp_dir: str) -> Generator[dict, None, None]: """Parses out the alert files from the hits.json and threats.json files Parameters ---------- temp_dir : str Folder which contains the expanded triage. Yields ------- Generator[dict, None, None] The next event found in the Triage. """ threats = None # We will always have 'hits.json' try: hits = json.load(open(f"{temp_dir}/hits.json", "r")) except Exception as e: logger.warning(f"Could not load JSON from hits.json, skipping alerts!") logger.debug(e) return if self.alert_files["threats.json"]: try: threats = json.load(open(f"{temp_dir}/threats.json", "r")) except Exception as e: logger.warning(f"Could not load JSON from threats.json, alert names may be UUIDs") logger.debug(e) else: logger.info(f"Could not find threats.json, alert names may be UUIDs") for alert in hits: # Introduce an alerting event type for HX. alert["event_type"] = "alertEvent" # If we have the threats file, convert the "threat_id" portion of the alert to # understandable values. if threats: threat = next( filter(lambda threat_entry: threat_entry["_id"] == alert["threat_id"], threats), None, ) if threat: logger.info( f"Matched up {alert['threat_id']} to {threat.get('display_name', threat['uri_name'])}" ) alert["_threat_data"] = threat # Add the time the alert happend alert["event_time"] = self._hx_time_to_epoch(alert["matched_at"]) alerting_event_type = alert["data"]["key"]["event_type"] # Strip the event type alerting_event = { k.replace(f"{alerting_event_type}/", ""): v for k, v in alert["data"]["values"].items() } alerting_event["event_time"] = self._hx_time_to_epoch(alerting_event["timestamp"]) alert["data"]["values"] = self._fix_missing_fields(alerting_event) yield alert
def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) logger.info("Created Generic Transformer.")
def __init__(self, evtx_log_file: str) -> None: self.file_path = evtx_log_file logger.info(f"Setting up WinEVTX for {self.file_path}")
def __init__(self, cuckoo_report: str) -> None: self.report = json.load(open(cuckoo_report, "r")) self.behavior = self.report["behavior"] self.processes: Dict[int, dict] = {} logger.info("Set up Cuckoo Sandbox")