def xml_records(filename): if OPTIONS.alternate: with Evtx(filename) as evtx: try: for xml, record in evtx_file_xml_view(evtx.get_file_header()): try: yield ef.to_lxml(xml), None except etree.XMLSyntaxError as e: yield xml, e except BinaryParser.OverrunBufferException as e: logging.error("Overrun Buffer Exception!") yield None, e except BinaryParser.ParseException as e: logging.error("Parse Exception!") yield None, e except Exception as e: # UnicodeDecodeError, AttributeError logging.error(e) yield None, e else: parser = PyEvtxParser(filename) try: for record in parser.records(): try: yield ef.to_lxml(record['data']), None except etree.XMLSyntaxError as e: yield record['data'], e except Exception as e: # UnicodeDecodeError, AttributeError, RuntimeError logging.error(e) yield None, e
def _read_evt_logs(func, logs_file: str, result_file: str, column_names: List[str]): global_start = time.time() start = time.time() _row_values = dict() if os.path.isfile(result_file): os.remove(result_file) with open(result_file, 'w', encoding='utf-8', newline='') as file: writer = csv.DictWriter(file, column_names) writer.writeheader() log = PyEvtxParser(logs_file) for index, record in enumerate(log.records()): _row_values.clear() _row_values = dict.fromkeys(column_names, 0) log_data = record['data'] func(log_data, _row_values) if IS_DEBUG: if index == 1000: break with open(result_file, 'a', encoding='utf-8', newline='') as file: writer = csv.DictWriter(file, column_names) writer.writerow(_row_values) if index % 10000 == 0: stop = time.time() print(f"{index}: {stop - start} сек.") start = time.time() print(f"затраченно времени: {time.time() - global_start} сек.")
def parse(self): parser = PyEvtxParser(self.eventfile) for record in parser.records_json(): rec = json.loads(record['data'])['Event'] data = {} # Common fields data['event.created'] = record.get( 'timestamp', rec['System']['TimeCreated']['#attributes']['SystemTime']) if isinstance(rec['System']['EventID'], dict): data['event.code'] = str(rec['System']['EventID']['#text']) else: data['event.code'] = str(rec['System']['EventID']) data['event.provider'] = rec['System']['Provider']['#attributes'][ 'Name'] data['event.dataset'] = rec['System']['Channel'] if 'Security' in rec['System']: try: data['user.id'] = rec['System']['Security']['#attributes'][ 'UserID'] except Exception: pass try: data['process.pid'] = rec['System']['Execution'][ '#attributes']['ProcessID'] data['process.thread.id'] = rec['System']['Execution'][ '#attributes']['ThreadID'] except Exception: pass # Events not defined in data_json if not data['event.code'] in self.data_json.keys( ) or data['event.provider'] != self.data_json[ data['event.code']]['provider']: # EventData, UserData are just reproduced as dictionaries if 'EventData' in rec: data['EventData'] = rec['EventData'] if 'UserData' in rec: data['UserData'] = rec['UserData'] yield data continue # Selected events data['description'] = self.data_json[ data['event.code']]["description"] for field in ['category', 'type', 'action']: if field in self.data_json[data['event.code']]: data['event.{}'.format(field)] = self.data_json[ data['event.code']][field] if 'path' not in self.data_json[data['event.code']].keys(): yield data continue # Extra fields for x, item in self.data_json[data['event.code']]['path'].items(): self.get_xpath_data(x, item, rec, data) yield data
def test_it_works_with_records(small_sample): parser = PyEvtxParser(small_sample) records = list(parser.records()) assert len(records) == 7 assert records[0]['event_record_id'] == 7 assert records[0]['timestamp'].endswith('UTC') assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_fails_nicely_on_close_files(small_sample): with open(small_sample, "rb") as o: parser = PyEvtxParser(o) with pytest.raises(OSError) as e: records = list(parser.records()) assert "closed file" in e.value.args[1]
def test_it_works_with_json(small_sample): parser = PyEvtxParser(small_sample) records = list(parser.records_json()) assert len(records) == 7 assert records[0]['event_record_id'] == 7 assert records[0]['timestamp'].endswith('UTC') assert json.loads(records[0]['data'])['Event']['System']['EventID'] == 4673
def parserecord(filename, num_items, logBufferSize, index, nodes, debug, support_queue, token=""): count = 0 JSONevents = "" logBufferLength = 0 count_postedrecord = 0 parser = PyEvtxParser(filename) for record in parser.records_json(): count = count + 1 logBufferLength = logBufferLength + 1 record = json.loads(record['data']) if isinstance(record['Event']['System']['EventID'], int): eventid = record['Event']['System']['EventID'] record['Event']['System']['EventID'] = {} record['Event']['System']['EventID']['#text'] = eventid # Same thing with their Data field try: if isinstance(record['Event']['EventData']['Data'], str): tmp = record['Event']['EventData']['Data'] record['Event']['EventData']['Data'] = {} record['Event']['EventData']['Data']['#text'] = tmp except KeyError: pass except TypeError: pass JSONevents = JSONevents + '{"index": {}}\n' JSONevents = JSONevents + json.dumps(record) + "\n" # Dump log buffer when full if logBufferLength >= int(logBufferSize): count_postedrecord = count_postedrecord + logBufferSize dump_batch(JSONevents, index, nodes, count_postedrecord, num_items, debug, support_queue, token) JSONevents = "" logBufferLength = 0 if logBufferLength > 0: count_postedrecord = count_postedrecord + logBufferLength dump_batch(JSONevents, index, nodes, count_postedrecord, num_items, debug, support_queue, token) logBufferLength = 0 JSONevents = "" return count_postedrecord
def test_it_supports_various_num_threads(small_sample): with open(small_sample, "rb") as o: parser = PyEvtxParser(o, number_of_threads=1) records = list(parser.records()) assert len(records) == 7 assert records[0]['event_record_id'] == 7 assert records[0]['timestamp'].endswith('UTC') assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_supports_various_ascii_codecs(small_sample): with open(small_sample, "rb") as o: parser = PyEvtxParser(o, ansi_codec="ascii") records = list(parser.records()) assert len(records) == 7 assert records[0]['event_record_id'] == 7 assert records[0]['timestamp'].endswith('UTC') assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_works_on_file_backed_object(small_sample): with open(small_sample, "rb") as o: parser = PyEvtxParser(o) records = list(parser.records()) assert len(records) == 7 assert records[0]['event_record_id'] == 7 assert records[0]['timestamp'].endswith('UTC') assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_works_on_io_object(small_sample): with open(small_sample, "rb") as o: r = o.read() parser = PyEvtxParser(io.BytesIO(r)) records = list(parser.records()) assert len(records) == 7 assert records[0]['event_record_id'] == 7 assert records[0]['timestamp'].endswith('UTC') assert '<EventID>4673</EventID>' in records[0]['data']
def run(self, target_file: TargetFile, result_of_previos_modules: dict) -> dict: events = [] self._filter_ids = self._get_filter_ids_from_params() parser = PyEvtxParser(target_file.get_path()) for record in parser.records(): try: parsed = xmltodict.parse(record["data"]) event = self._get_info(parsed, target_file) if event is not None: events.append(event) except: pass piped = utils.pipe_to_another_output(self._params, events) return {"n_events": len(events)} if piped else {"events": events}
def runUsingBindings(self, file): """ Convert EVTX to JSON using evtx_dump bindings (slower) Drop resulting JSON files in a tmp folder. """ try: filepath = Path(file) filename = filepath.name parser = PyEvtxParser(str(filepath)) with open( f"{self.tmpDir}/{str(filename)}-{self.randString()}.json", "w") as f: for record in parser.records_json(): f.write(f'{json.dumps(json.loads(record["data"]))}\n') except Exception as e: self.logger.error(f"{Fore.RED} [-] {e}")
def MatchLogFile(self, filePath): parser = PyEvtxParser(filePath) if self.logger: self.logger.info( f"Statrted processing the Event log '{filePath}'.") for record in parser.records_json(): try: data = json.loads(record["data"]) event = Event(data) self.matchAll(event) except Exception as e: if self.logger: self.logger.error(e, exc_info=True) if self.logger: self.logger.info( f"Finished processing the Event log '{filePath}'.")
def asyncGenLogs(file): EVTx = {} json_data_dir = createDir('web\json_data') print('EVTxFILE:', file) fname = filename(file) _df = pd.DataFrame() parser = PyEvtxParser(file) i = 0 # Register counter for record in parser.records_json(): data = json.loads(record['data'])['Event'] readXmlObj(data, EVTx) try: df = pd.DataFrame.from_dict(EVTx).fillna(0) _df = _df.append(df) i += 1 except Exception as e: pass EVTx = {} if i > (REG_COUNT_LIMIT - 1): break if not _df.empty: if REGEX_PATERN: try: _df = _df[_df.apply(lambda x: x.astype(str).str.contains( REGEX_PATERN, case=False, na=False)).any(axis=1)] except Exception as e: # error(e) #verbose pass _df = _df.set_index('EventRecordID').sort_values(by=['EventRecordID'], ascending=False) _df.to_json(f'{json_data_dir}\\{md5(fname)}.json', orient="table") r, c = _df.shape eel.feedback(f'Lines: {r}, Columns: {c}', md5(file)) else: pd.DataFrame(['Empty'], columns=['File' ]).to_json(f'{json_data_dir}\\{md5(fname)}.json', index=False, orient="table") eel.feedback(f'Empty', md5(file))
def Evtx_hunt(files, str_regex, input_timzone, output): for file in files: print("Analyzing " + file) try: parser = PyEvtxParser(file) except: print("Issue analyzing " + file + "\nplease check if its not corrupted") continue try: rex = re.compile(str_regex, re.IGNORECASE) for record in parser.records(): EventID = EventID_rex.findall(record['data']) if len(EventID) > 0: Computer = Computer_rex.findall(record['data']) Channel = Channel_rex.findall(record['data']) if len(Channel) > 0: channel = Channel[0] else: channel = " " #print(record['data']) # if record['data'].lower().find(str_regex.lower())>-1: if rex.findall(record['data']): #print("EventID : "+EventID[0]+" , Data : "+record['data']) Hunting_events[0]['timestamp'].append( datetime.timestamp( isoparse( parse(record["timestamp"]).astimezone( input_timzone).isoformat()))) Hunting_events[0]['Date and Time'].append( parse(record["timestamp"]).astimezone( input_timzone).isoformat()) Hunting_events[0]['Channel'].append(channel) Hunting_events[0]['Event ID'].append(EventID[0]) Hunting_events[0]['Computer'].append(Computer[0]) Hunting_events[0]['Original Event Log'].append( str(record['data']).replace("\r", " ").replace( "\n", " ")) except Exception as e: print("issue searching log : " + record['data'] + "\n Error : " + print(e)) hunt_report(output)
class Evtx2es(object): def __init__(self, input_path: Path) -> None: self.path = input_path self.parser = PyEvtxParser(self.path.open(mode="rb")) def gen_records(self, shift: Union[str, datetime], multiprocess: bool, chunk_size: int) -> Generator: """Generates the formatted Eventlog records chunks. Args: multiprocess (bool): Flag to run multiprocessing. chunk_size (int): Size of the chunk to be processed for each process. Yields: Generator: Yields List[dict]. """ gen_path = iter(lambda: str(self.path), None) gen_shift = iter(lambda: shift, None) if multiprocess: with Pool(cpu_count()) as pool: results = pool.starmap_async( process_by_chunk, zip( generate_chunks(chunk_size, self.parser.records_json()), gen_path, gen_shift, )) yield list(chain.from_iterable(results.get(timeout=None))) else: buffer: List[List[dict]] = list() for records in generate_chunks(chunk_size, self.parser.records_json()): if chunk_size <= len(buffer): yield list(chain.from_iterable(buffer)) buffer.clear() else: buffer.append( process_by_chunk(records, gen_path, gen_shift)) else: yield list(chain.from_iterable(buffer))
def load_events(log_file_name): """ Opens Sysmon logs as a readable file, and turns the events into dictionaries :param log_file_name: Sysmon evtx or xml log to be opened :return: dict of the Sysmon event log """ try: with open(log_file_name, "r", encoding='utf-8', errors='ignore') as fp: magic = fp.read(7) fp.seek(0) if magic == 'ElfFile': # log is evtx type parser = PyEvtxParser(log_file_name) dictrecords = [json.loads(rec['data']) for rec in parser.records_json()] return dictrecords, 'evtx' elif magic == '<Events': return xmltodict.parse(fp.read()), 'xml' else: raise Exception('Invalid File magic') except ExpatError: raise KeyError("Error: Format error in the Event log file")
def read_evt_logs(func, logs_file: str = 'data/Security.evtx', result_file: str = 'data/column_names_utf_8'): global_start = time.time() start = time.time() stop = 0 log = PyEvtxParser(logs_file) for index, record in enumerate(log.records()): # if index > 0: # break log_data = record['data'] func(log_data) if index % 10000 == 0: stop = time.time() print(f"{index}: {stop - start} сек.") start = time.time() try: with open(result_file, 'w', encoding='utf-8') as file: json.dump(column_names, file) except Exception as ex: print(ex) print(f"затраченно времени: {time.time() - global_start} сек.")
class EvtxViewModel(QAbstractItemModel): def __init__(self, filename): super(QAbstractItemModel, self).__init__() self.__filename = filename self.__chunks = 0 self.__columns = [ ('TimeCreated', lambda r: r.timestamp), ('Provider', lambda r: r.Provider), ('EventID', lambda r: r.EventID), ('EventData', lambda r: r.Level) ] for idx in range(0, len(self.__columns)): self.setHeaderData(idx, Qt.Horizontal, self.__columns[0][0], Qt.DisplayRole) self.__records = dict() self.__record_ids = list() self.load_data() def load_data(self): self.__parser = PyEvtxParser(self.__filename) for record in self.__parser.records(): self.__records[record["event_record_id"]] = EventRecord(record) self.__record_ids.append(record["event_record_id"]) self.__record_ids.sort() def rowCount(self, parent): return len(self.__record_ids) def columnCount(self, parent): return len(self.__columns) def index(self, row: int, column: int, parent: QModelIndex = ...) -> QModelIndex: return self.createIndex(row, column) def data(self, index: QModelIndex, role: int = ...) -> typing.Any: if role == Qt.DisplayRole: row = index.row() record_id = self.__record_ids[row] record = self.__records[record_id] column = self.__columns[index.column()] return column[1](record) else: return QVariant() def flags(self, index: QModelIndex) -> Qt.ItemFlags: return Qt.ItemIsEnabled | Qt.ItemNeverHasChildren def parent(self, qmodelindex=None): return QModelIndex()
def test_it_returns_error_when_iterating_twice(small_sample): parser = PyEvtxParser(small_sample) _ = list(parser.records()) with pytest.raises(RuntimeError): parser.records()
def test_it_returns_error_when_using_next_on_parser(small_sample): parser = PyEvtxParser(small_sample) with pytest.raises(NotImplementedError): next(parser)
def parser_simple(path, opath=None): import time cursor = sql_connection() sql_initialitation(cursor) parser = PyEvtxParser(path) connections_key = 100000000 thread_key = 0 files_inserted = [] full_process_inserted = [] pipes_inserted = [] threads_inserted = [] start = time.time() for record in parser.records_json(): event = json.loads(record['data']) # Process Creation if event["Event"]["System"]["EventID"] == 1: try: if event["Event"]["EventData"]["ProcessGuid"] not in full_process_inserted: #print("Full inserted") query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image","IntegrityLevel",' \ '"TerminalSessionId", "User")' \ " VALUES ('{}','{}','{}','{}','{}', '{}')" \ ' ON CONFLICT ("ProcessGuid") DO UPDATE SET "IntegrityLevel" = ' \ 'EXCLUDED."IntegrityLevel", "TerminalSessionId"' \ ' = EXCLUDED."TerminalSessionId", "User" = EXCLUDED."User";'.format( (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["ProcessId"], event["Event"]["EventData"]["Image"], event["Event"]["EventData"]["IntegrityLevel"], event["Event"]["EventData"]["TerminalSessionId"], event["Event"]["EventData"]["User"]) cursor.execute(query_process) full_process_inserted.append(event["Event"]["EventData"]["ProcessGuid"]) except Exception as e: logger.error("Error query_process 1: " + str(e) + " Event: " + str(event["Event"])) try: if event["Event"]["EventData"]["Image"] not in files_inserted: if "OriginalFileName" in event["Event"]["EventData"]: original = ",'" + str(event["Event"]["EventData"]["OriginalFileName"]) + "'" key = ',"OriginalFileName"' else: original = "" key = "" except Exception as e: logger.error("Error query_file 1: " + str(e) + " Event: " + str(event["Event"])) try: query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["ParentImage"]).lower(), event["Event"]["EventData"]["ParentProcessId"], event["Event"]["EventData"]["ParentImage"]) cursor.execute(query_pprocess) except Exception as e: logger.error("Error query_pprocess_exist 1: " + str(e) + " Event: " + str(query_pprocess)) try: if event["Event"]["EventData"]["CommandLine"]: event["Event"]["EventData"]["CommandLine"] = str( event["Event"]["EventData"]["CommandLine"]).replace("'", "\"") if event["Event"]["EventData"]["CurrentDirectory"]: event["Event"]["EventData"]["CurrentDirectory"] = str( event["Event"]["EventData"]["CurrentDirectory"]).replace("'", "\"") query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \ '"ExtraInfo","ExtraInfo2")' \ " VALUES ('{}','{}','{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateProcess", (event["Event"]["EventData"]["ParentImage"]).lower(), event["Event"]["EventData"]["LogonGuid"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["CommandLine"], event["Event"]["EventData"]["CurrentDirectory"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 1: " + str(e) + " Event: " + str(query_action)) try: query_user = '******' \ " ('{}','{}','{}',{}) ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["LogonGuid"], event["Event"]["EventData"]["User"], event["Event"]["EventData"]["LogonId"], event["Event"]["EventData"]["TerminalSessionId"]) cursor.execute(query_user) except Exception as e: logger.error("Error 1: " + str(e) + " Event: " + str(query_user)) # File creation time changed if event["Event"]["System"]["EventID"] == 2: logger.info("ToDo") # Network connection if event["Event"]["System"]["EventID"] == 3: try: connections_key = str(event["Event"]["EventData"]["SourceIp"])+str(event["Event"]["EventData"]["DestinationIp"]) query_connection = 'INSERT INTO public."Connections" ("ConnectionId","Protocol","SourceIp","SourceHostname",' \ '"SourcePort","DestinationIsIpv6","DestinationIp","DestinationHostname","DestinationPort") ' \ "VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format( connections_key, event["Event"]["EventData"]["Protocol"], event["Event"]["EventData"]["SourceIp"], event["Event"]["EventData"]["SourceHostname"], event["Event"]["EventData"]["SourcePort"], event["Event"]["EventData"]["DestinationIsIpv6"], event["Event"]["EventData"]["DestinationIp"], event["Event"]["EventData"]["DestinationHostname"], event["Event"]["EventData"]["DestinationPort"]) cursor.execute(query_connection) except Exception as e: logger.error("Error query_connection 3: " + str(e) + " Event: " + str(query_connection)) try: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \ '"ExtraInfo")' \ " VALUES ('{}','{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateConnection", (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["User"], connections_key, event["Event"]["EventData"]["Initiated"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 3: " + str(e) + " Event: " + str(query_action)) try: insert_process(cursor, event) except Exception as e: logger.error("Error query_pprocess_exist 3: " + str(e) + " Event: " + str(event["Event"])) # Process Terminated if event["Event"]["System"]["EventID"] == 5: try: # Destination Process query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["ProcessId"], event["Event"]["EventData"]["Image"]) cursor.execute(query_process) except Exception as e: logger.error("Error query_sprocess 5: " + str(e) + " Event: " + str(query_process)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "ProcessTerminated", (event["Event"]["EventData"]["Image"]).lower(), (event["Event"]["EventData"]["Image"]).lower()) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 5: " + str(e) + " Event: " + str(query_action)) # Kernel driver loaded if event["Event"]["System"]["EventID"] == 6: logger.info("ToDo") # Image loaded if event["Event"]["System"]["EventID"] == 7: try: # Process insert_process(cursor, event) except Exception as e: logger.error("Error query_sprocess 7: " + str(e) + " Event: " + str(query_sprocess)) try: # File if "Description" not in event["Event"]["EventData"]: event["Event"]["EventData"]["Description"] = "" if "OriginalFileName" not in event["Event"]["EventData"]: event["Event"]["EventData"]["OriginalFileName"] = "" query_file = 'INSERT INTO public."Files" ("Filename","FileVersion","Description","Product","Company","OriginalFileName","Hashes","Signed","Signature","SignatureStatus") ' \ " VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT" \ ' ("Filename") DO UPDATE SET' \ ' "FileVersion" = EXCLUDED."FileVersion", "Description" = EXCLUDED."Description",' \ ' "Product" = EXCLUDED."Product", "Company" = EXCLUDED."Company",' \ '"OriginalFileName" = EXCLUDED."OriginalFileName","Hashes" = EXCLUDED."Hashes"' \ ',"Signed" = EXCLUDED."Signed","Signature" = EXCLUDED."Signature",' \ '"SignatureStatus" = EXCLUDED."SignatureStatus";'.format( "f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(), event["Event"]["EventData"]["FileVersion"], event["Event"]["EventData"]["Description"], event["Event"]["EventData"]["Product"], event["Event"]["EventData"]["Company"], event["Event"]["EventData"]["OriginalFileName"], event["Event"]["EventData"]["Hashes"], event["Event"]["EventData"]["Signed"], event["Event"]["EventData"]["Signature"], event["Event"]["EventData"]["SignatureStatus"]) cursor.execute(query_file) except Exception as e: logger.error("Error query_file 7: " + str(e) + " Event: " + str(query_file)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "LoadImage", "f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(), (event["Event"]["EventData"]["Image"]).lower()) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 7: " + str(e) + " Event: " + str(query_action)) # Create Remote Thread if event["Event"]["System"]["EventID"] == 8: try: # Source Process query_sprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image")' \ " VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;"\ .format((event["Event"]["EventData"]["SourceImage"]).lower(), event["Event"]["EventData"]["SourceProcessId"], (event["Event"]["EventData"]["SourceImage"]).lower()) cursor.execute(query_sprocess) except Exception as e: logger.error("Error query_sprocess 8: " + str(e) + " Event: " + str(query_sprocess)) try: # Target Process query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["TargetProcessId"], (event["Event"]["EventData"]["TargetImage"]).lower()) cursor.execute(query_tprocess) except Exception as e: logger.error("Error query_tprocess 8: " + str(e) + " Event: " + str(query_tprocess)) try: # Thread thread_key = str(event["Event"]["EventData"]["TargetProcessGuid"]) + ":" + str(event["Event"]["EventData"]["NewThreadId"]) query_thread = 'INSERT INTO public."Threads" ("ThreadId","ThreadNId","ProcessGuid","StartAddress",' \ '"StartModule", "StartFunction")' \ " VALUES ('{}','{}','{}','{}','{}','{}') ON CONFLICT " \ '("ThreadId") DO UPDATE SET "StartAddress" = ' \ 'EXCLUDED."StartAddress", "StartModule"' \ ' = EXCLUDED."StartModule", "StartFunction"' \ ' = EXCLUDED."StartFunction";'.format( thread_key, event["Event"]["EventData"]["NewThreadId"], (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["StartAddress"], event["Event"]["EventData"]["StartModule"], event["Event"]["EventData"]["StartFunction"]) cursor.execute(query_thread) except Exception as e: logger.error("Error query_thread 8: " + str(e) + " Event: " + str(query_thread)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateRemoteThread", (event["Event"]["EventData"]["SourceImage"]).lower(), thread_key) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 8: " + str(e) + " Event: " + str(query_action)) # Raw access read if event["Event"]["System"]["EventID"] == 9: logger.info("ToDo") # Process Access if event["Event"]["System"]["EventID"] == 10: try: query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["SourceImage"]).lower(), event["Event"]["EventData"]["SourceProcessId"], (event["Event"]["EventData"]["SourceImage"]).lower()) cursor.execute(query_pprocess) except Exception as e: logger.error("Error query_Pprocess 10: " + str(e) + " Event: " + str(query_pprocess)) try: query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["TargetProcessId"], (event["Event"]["EventData"]["TargetImage"]).lower()) cursor.execute(query_tprocess) except Exception as e: logger.error("Error query_tprocess 10: " + str(e) + " Event: " + str(query_tprocess)) try: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId",' \ '"ExtraInfo","ExtraInfo2")' \ " VALUES ('{}','{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "ProcessAccess", (event["Event"]["EventData"]["SourceImage"]).lower(), (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["GrantedAccess"], event["Event"]["EventData"]["CallTrace"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 10: " + str(e) + " Event: " + str(query_action)) # File create if event["Event"]["System"]["EventID"] == 11: # Create File try: # Process insert_process(cursor, event) except Exception as e: print("Error query_sprocess 11: " + str(e) + " Event: " + str(event["Event"])) try: # File query_file = 'INSERT INTO public."Files" ("Filename","CreationUtcTime") ' \ "VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format( "f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower(), event["Event"]["EventData"]["CreationUtcTime"]) cursor.execute(query_file) except Exception as e: logger.error("Error query_file 11: " + str(e) + " Event: " + str(query_file)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateFile", (event["Event"]["EventData"]["Image"]).lower(), "f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower()) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 11: " + str(e) + " Event: " + str(query_action)) # Registry Key Operation if event["Event"]["System"]["EventID"] == 12 or event["Event"]["System"]["EventID"] == 13 \ or event["Event"]["System"]["EventID"] == 14: if event["Event"]["EventData"]["TargetObject"]: event["Event"]["EventData"]["TargetObject"] = str( event["Event"]["EventData"]["TargetObject"]).replace("'", "\"") try: # Process insert_process(cursor, event) except Exception as e: logger.error("Error query_process 12-13-14: " + str(e) + " Event: " + str(event["Event"])) try: # RegistryKey if event["Event"]["System"]["EventID"] == 13: if event["Event"]["EventData"]["Details"]: event["Event"]["EventData"]["Details"] = str( event["Event"]["EventData"]["Details"]).replace("'", "\"") query_key = 'INSERT INTO public."RegistryKeys" ("Key","Details") ' \ "VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format( event["Event"]["EventData"]["TargetObject"], event["Event"]["EventData"]["Details"]) else: query_key = 'INSERT INTO public."RegistryKeys" ("Key") ' \ "VALUES ('{}') ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["TargetObject"]) cursor.execute(query_key) except Exception as e: logger.error("Error query_key 12-13-14: " + str(e) + " Event: " + str(query_key)) try: # Action if "SetValue" in event["Event"]["EventData"]["EventType"]: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId","ExtraInfo")' \ " VALUES ('{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "RegistryKey-" + event["Event"]["EventData"]["EventType"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["TargetObject"], event["Event"]["EventData"]["Details"]) else: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "RegistryKey-" + event["Event"]["EventData"]["EventType"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["TargetObject"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 12-13-14: " + str(e) + " Event: " + str(query_action)) # File create stream hash if event["Event"]["System"]["EventID"] == 15: logger.info("ToDo") # File create stream hash if event["Event"]["System"]["EventID"] == 15: logger.info("ToDo") # Pipe event if event["Event"]["System"]["EventID"] == 17 or event["Event"]["System"]["EventID"] == 18: try: insert_process(cursor, event) except Exception as e: print("Error insert_process 17-18: " + str(e) + " Event: " + str(event["Event"])) try: # Pipe if event["Event"]["EventData"]["PipeName"] not in pipes_inserted: query_pipe = 'INSERT INTO public."Pipes" ("PipeName") ' \ "VALUES ('{}');".format( event["Event"]["EventData"]["PipeName"]) cursor.execute(query_pipe) pipes_inserted.append(event["Event"]["EventData"]["PipeName"]) except Exception as e: logger.error("Error query_file 17-18: " + str(e) + " Event: " + str(query_pipe)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], event["Event"]["EventData"]["EventType"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["PipeName"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 17-18: " + str(e) + " Event: " + str(query_action)) # WMI event if event["Event"]["System"]["EventID"] == 19 or event["Event"]["System"]["EventID"] == 20 \ or event["Event"]["System"]["EventID"] == 21: logger.info("ToDo") # DNS if event["Event"]["System"]["EventID"] == 22: try: insert_process(cursor, event) except Exception as e: logger.error("Error insert_process 22: " + str(e) + " Event: " + str(event["Event"])) try: # Query query_dnsquery = 'INSERT INTO public."DNSQuery" ("QueryName") ' \ "VALUES ('{}') ON CONFLICT DO NOTHING;".format( event["Event"]["EventData"]["QueryName"]) cursor.execute(query_dnsquery) except Exception as e: logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsquery)) try: # Resolution query_dnsresolution = 'INSERT INTO public."DNSResolution" ("UtcTime","QueryName","QueryStatus","QueryResults") ' \ "VALUES ('{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format( event["Event"]["EventData"]["UtcTime"], event["Event"]["EventData"]["QueryName"], event["Event"]["EventData"]["QueryStatus"], event["Event"]["EventData"]["QueryResults"]) cursor.execute(query_dnsresolution) except Exception as e: logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsresolution)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}') ;".format( event["Event"]["EventData"]["UtcTime"], "DnsRequest", (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["QueryName"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 22: " + str(e) + " Event: " + str(query_action)) cursor.close() end = time.time() logger.info("Time to process file %s seconds ---" % (end - start))
def load_data(self): self.__parser = PyEvtxParser(self.__filename) for record in self.__parser.records(): self.__records[record["event_record_id"]] = EventRecord(record) self.__record_ids.append(record["event_record_id"]) self.__record_ids.sort()
def __init__(self, filepath: str) -> None: self.path = Path(filepath) self.parser = PyEvtxParser(self.path.open(mode='rb'))
class Evtx2es(object): def __init__(self, filepath: str) -> None: self.path = Path(filepath) self.parser = PyEvtxParser(self.path.open(mode='rb')) def gen_json(self, size: int) -> Generator: buffer: List[dict] = [] for record in self.parser.records_json(): record['data'] = json.loads(record.get('data')) eventid_field = record.get('data').get('Event').get('System').get( 'EventID') if type(eventid_field) is dict: record['data']['Event']['System'][ 'EventID'] = eventid_field.get('#text') try: status = record.get('data').get('Event').get('EventData').get( 'Status') record['data']['Event']['EventData']['Status'] = None except Exception: pass # Convert data according to ECS (sort of) # First copy system fields record['winlog'] = { 'channel': record['data']['Event']['System']['Channel'], 'computer_name': record['data']['Event']['System']['Computer'], 'event_id': record['data']['Event']['System']['EventID'], 'opcode': record['data']['Event']['System'].get('Opcode'), 'provider_guid': record['data']['Event']['System']['Provider'] ['#attributes'].get('Guid'), 'provider_name': record['data']['Event']['System']['Provider']['#attributes'] ['Name'], 'record_id': record['data']['Event']['System']['EventRecordID'], 'task': record['data']['Event']['System']['Task'], 'version': record['data']['Event']['System'].get('Version'), } try: record['winlog']['process'] = { 'pid': record['data']['Event']['System']['Execution'] ['#attributes']['ProcessID'], 'thread_id': record['data']['Event']['System']['Execution'] ['#attributes']['ThreadID'], } except KeyError: pass record.update({ 'log': { 'file': { 'name': str(self.path) } }, 'event': { 'code': record['winlog']['event_id'], 'created': record['data']['Event']['System']['TimeCreated'] ['#attributes']['SystemTime'], } }) record['@timestamp'] = record['event']['created'] # Move event attributes to ECS location record['winlog']['event_data'] = record['data']['Event'].get( 'EventData', dict()) del record['data'] if record['winlog']['event_data'] is None or len( record['winlog'] ['event_data']) == 0: # remove event_data fields if empty del record['winlog']['event_data'] else: for k, v in record['winlog']['event_data'].items(): # Normalize some known problematic fields with values switching between integers and strings with hexadecimal notation to integers if k in ('ProcessId') and type(v) == str: if v.startswith("0x"): record['winlog']['event_data'][k] = int(v, 16) else: try: record['winlog']['event_data'][k] = int(v) except ValueError: record['winlog']['event_data'][k] = 0 # Maximum limit of numeric values in Elasticsearch if type(v) is int: if v < -2**63: record['winlog']['event_data'][k] = -2**63 elif v > 2**63 - 1: record['winlog']['event_data'][k] = 2**63 - 1 buffer.append(record) if len(buffer) >= size: yield buffer buffer.clear() else: yield buffer
def ProcessPrivateRules(self, logspath): if self.logger: self.logger.info( f"Starting processing private rules on the log/s in '{logspath}' ..." ) for pubrule in self.PublicRulesContainsPrivateRules: triggered = None privRules = [] privRulesChannels = [] TriggeredEvents = {} for privrulename in pubrule.include.get("rule"): for rule in self.ruleSet: if rule.name == privrulename: privRules.append(rule) privRulesChannels.append(rule.channel.lower()) for filePath in self.LogsToProcess: parser = PyEvtxParser(filePath) for record in parser.records_json(): try: data = json.loads(record["data"]) event = Event(data) if event.Channel.lower() in privRulesChannels: for prirule in privRules: if self.match(prirule, event): if triggered == None: triggered = True triggered = triggered and True if not TriggeredEvents.get(prirule): TriggeredEvents[prirule] = [event] else: TriggeredEvents[prirule].append(event) else: break except (OSError, KeyError) as e: if self.logger: self.logger.error(e, exc_info=True) continue except Exception as e: if self.logger: self.logger.error(e, exc_info=True) if len(TriggeredEvents) != len(privRules): return False TriggeredEventsList = [] for key, val in TriggeredEvents.items(): TriggeredEventsList.append(val) TriggeredEventsWithinTheSpecifiedTime = self.ProcessTimeBetweenLogs( list(itertools.product(*TriggeredEventsList)), int(pubrule.include.get("if").get("within"))) if TriggeredEventsWithinTheSpecifiedTime: for EventSet in TriggeredEventsWithinTheSpecifiedTime: recordIDs = [] triggeredEventsData = {} privRuleNames = pubrule.include.get("rule") privateRules = [] for r in self.ruleSet: if r.name in privRuleNames: privateRules.append(r) for r in privateRules: for e in EventSet: if r.channel == e.Channel: if r.returns: fields = {} for field in r.returns: fields.update( {field: e.EventData.get(field)}) triggeredEventsData[r.name] = fields else: triggeredEventsData[r.name] = e.RawRecord for event in EventSet: recordIDs.append(event.EventRecordID) data = [ event.TimeCreatedSystemTime, recordIDs, pubrule.name, pubrule.score, pubrule.description, pubrule.reference, [], triggeredEventsData ] self.Queue.put( Alert(event, pubrule, [], privateRule=True, record=data)) else: return False
def test_it_returns_error_on_non_existing_path(): with pytest.raises(FileNotFoundError): parser = PyEvtxParser("non_existing")
def parseevtx(infile, outdir, coldelimiter, jsonfile): outfilenametmp = os.path.basename(infile) + ".csv" if jsonfile == 1: jsonoutfilenametmp = os.path.basename(infile) + ".json" drive, outfiledir = os.path.splitdrive(os.path.dirname(infile)) #Make Dir harddir = outdir + os.sep + outfiledir outfilename = harddir + os.sep + outfilenametmp os.makedirs(harddir, exist_ok=True) print(f'CVS Out Info : {outfilename}') if jsonfile == 1: jsonoutfilename = harddir + os.sep + jsonoutfilenametmp print(f'JSON Out Info: {jsonoutfilename}') jsonoutfile = open(jsonoutfilename, 'w') # #1 #2 #3 #4 #5 #6 #7 #8 #9 #10 #11 header = "'EventID'|'ComputerName'|'TimeCreated'|'Security SID'|'UserID'|'Address'|'Script/IP:Port'|'Payload/Path/Correlation Activity ID'|'Event'|'Channel (EVTX Log)'|'SRC File'\n" #create file outfile = open(outfilename, 'w') outfile.write( "Please note that depending on the CSV deliminater used, the fields may not line up.\n" ) outfile.write(header) parser = PyEvtxParser(infile) print(f'Working on: {infile}') for record in parser.records_json(): subrec = json.loads(record["data"]) lineout = "" try: eventid = subrec['Event']['System']['EventID'] #['#text'] except: eventid = subrec['Event']['System']['EventID']['#text'] channel = subrec['Event']['System']['Channel'] computer = subrec['Event']['System']['Computer'] systemtime = subrec['Event']['System']['TimeCreated']['#attributes'][ 'SystemTime'] #print(subrec['Event']['UserData']['EventXML'].keys()) if jsonfile == 1: if channel == "Windows PowerShell" or channel == "Microsoft-Windows-WinRM/Operational" or channel == "Microsoft-Windows-PowerShell/Operational" or channel == "Security" or channel == "System": jsonoutfile.write(printjson(subrec)) jsonoutfile.write("\n") #print(f'EID: {eventid}') #-------------------------------------------------------------------------------------------------------------------------------- #Windows PowerShell.evtx if eventid == 400 or eventid == 403 or eventid == 600: if channel == "Windows PowerShell": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['Data']['#text']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- #Microsoft-Windows-WinRM/Operational.evtx if eventid == 6: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Connecting remotely to: " + subrec[ 'Event']['EventData'][ 'connection'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 81: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Operation Name: " + subrec['Event'][ 'EventData']['operationName'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 82: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Operation: " + subrec['Event'][ 'EventData'][ 'operation'] + " -- Resource URI: " + subrec[ 'Event']['EventData'][ 'resourceURI'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 134: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Operation: " + subrec['Event'][ 'EventData']['operationName'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) # Need an example log related to powershell; now it prints out all 142 events if eventid == 142: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'OperatinName: " + subrec['Event'][ 'EventData'][ 'operationName'] + " -- Error Code :'" + str( subrec['Event']['EventData'] ['errorCode']) + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 169: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "'" + subrec['Event']['EventData'][ 'username'] + "'" + coldelimiter lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Authentication Mechanism: " + subrec[ 'Event']['EventData'][ 'authenticationMechanism'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- #Microsoft-Windows-PowerShell/Operational.evtx if eventid == 4100: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['ContextInfo']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "'" + subrec['Event']['EventData'][ 'Payload'] + "'" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 4103: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) ScriptBlockText address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['ContextInfo']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "'" + subrec['Event']['EventData'][ 'Path'] + "'" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 4104: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['ScriptBlockText']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "'" + subrec['Event']['EventData'][ 'Path'] + "'" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 40961: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- #Security if eventid == 4688: if channel == "Security": try: if re.search( 'powershell', subrec['Event']['EventData']['NewProcessName'], re.IGNORECASE): #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['EventData'][ 'SubjectDomainName'] + "\\" + subrec['Event'][ 'EventData']['SubjectUserName'] + " (" + subrec[ 'Event']['EventData'][ 'SubjectUserSid'] + ")'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 try: tmpstr = subrec['Event']['EventData'][ 'CommandLine'] try: tmpppn = subrec['Event']['EventData'][ 'ParentProcessName'] except: tmpppn = '' if tmpstr: lineout = lineout + "'" + tmpppn + " -> " + subrec[ 'Event']['EventData'][ 'CommandLine'] + "'" + coldelimiter #7 else: lineout = lineout + "'" + tmpppn + " -> " + subrec[ 'Event']['EventData'][ 'NewProcessName'] + "'" + coldelimiter #7 except: lineout = lineout + "'" + tmpppn + " -> " + subrec[ 'Event']['EventData'][ 'NewProcessName'] + "'" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- if eventid == 7030 or eventid == 7040 or eventid == 7045: if channel == "System": try: printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) print("end") outfile.close() if jsonfile == 1: jsonoutfile.close()
def test_it_fails_on_file_opened_as_text(small_sample): with pytest.raises(OSError) as e: with open(small_sample, "rt") as o: parser = PyEvtxParser(o) assert "decode byte" in e.value.args[1]