def parse(self): parser = PyEvtxParser(self.eventfile) for record in parser.records_json(): rec = json.loads(record['data'])['Event'] data = {} # Common fields data['event.created'] = record.get( 'timestamp', rec['System']['TimeCreated']['#attributes']['SystemTime']) if isinstance(rec['System']['EventID'], dict): data['event.code'] = str(rec['System']['EventID']['#text']) else: data['event.code'] = str(rec['System']['EventID']) data['event.provider'] = rec['System']['Provider']['#attributes'][ 'Name'] data['event.dataset'] = rec['System']['Channel'] if 'Security' in rec['System']: try: data['user.id'] = rec['System']['Security']['#attributes'][ 'UserID'] except Exception: pass try: data['process.pid'] = rec['System']['Execution'][ '#attributes']['ProcessID'] data['process.thread.id'] = rec['System']['Execution'][ '#attributes']['ThreadID'] except Exception: pass # Events not defined in data_json if not data['event.code'] in self.data_json.keys( ) or data['event.provider'] != self.data_json[ data['event.code']]['provider']: # EventData, UserData are just reproduced as dictionaries if 'EventData' in rec: data['EventData'] = rec['EventData'] if 'UserData' in rec: data['UserData'] = rec['UserData'] yield data continue # Selected events data['description'] = self.data_json[ data['event.code']]["description"] for field in ['category', 'type', 'action']: if field in self.data_json[data['event.code']]: data['event.{}'.format(field)] = self.data_json[ data['event.code']][field] if 'path' not in self.data_json[data['event.code']].keys(): yield data continue # Extra fields for x, item in self.data_json[data['event.code']]['path'].items(): self.get_xpath_data(x, item, rec, data) yield data
def test_it_works_with_json(small_sample): parser = PyEvtxParser(small_sample) records = list(parser.records_json()) assert len(records) == 7 assert records[0]['event_record_id'] == 7 assert records[0]['timestamp'].endswith('UTC') assert json.loads(records[0]['data'])['Event']['System']['EventID'] == 4673
def parserecord(filename, num_items, logBufferSize, index, nodes, debug, support_queue, token=""): count = 0 JSONevents = "" logBufferLength = 0 count_postedrecord = 0 parser = PyEvtxParser(filename) for record in parser.records_json(): count = count + 1 logBufferLength = logBufferLength + 1 record = json.loads(record['data']) if isinstance(record['Event']['System']['EventID'], int): eventid = record['Event']['System']['EventID'] record['Event']['System']['EventID'] = {} record['Event']['System']['EventID']['#text'] = eventid # Same thing with their Data field try: if isinstance(record['Event']['EventData']['Data'], str): tmp = record['Event']['EventData']['Data'] record['Event']['EventData']['Data'] = {} record['Event']['EventData']['Data']['#text'] = tmp except KeyError: pass except TypeError: pass JSONevents = JSONevents + '{"index": {}}\n' JSONevents = JSONevents + json.dumps(record) + "\n" # Dump log buffer when full if logBufferLength >= int(logBufferSize): count_postedrecord = count_postedrecord + logBufferSize dump_batch(JSONevents, index, nodes, count_postedrecord, num_items, debug, support_queue, token) JSONevents = "" logBufferLength = 0 if logBufferLength > 0: count_postedrecord = count_postedrecord + logBufferLength dump_batch(JSONevents, index, nodes, count_postedrecord, num_items, debug, support_queue, token) logBufferLength = 0 JSONevents = "" return count_postedrecord
class Evtx2es(object): def __init__(self, input_path: Path) -> None: self.path = input_path self.parser = PyEvtxParser(self.path.open(mode="rb")) def gen_records(self, shift: Union[str, datetime], multiprocess: bool, chunk_size: int) -> Generator: """Generates the formatted Eventlog records chunks. Args: multiprocess (bool): Flag to run multiprocessing. chunk_size (int): Size of the chunk to be processed for each process. Yields: Generator: Yields List[dict]. """ gen_path = iter(lambda: str(self.path), None) gen_shift = iter(lambda: shift, None) if multiprocess: with Pool(cpu_count()) as pool: results = pool.starmap_async( process_by_chunk, zip( generate_chunks(chunk_size, self.parser.records_json()), gen_path, gen_shift, )) yield list(chain.from_iterable(results.get(timeout=None))) else: buffer: List[List[dict]] = list() for records in generate_chunks(chunk_size, self.parser.records_json()): if chunk_size <= len(buffer): yield list(chain.from_iterable(buffer)) buffer.clear() else: buffer.append( process_by_chunk(records, gen_path, gen_shift)) else: yield list(chain.from_iterable(buffer))
def runUsingBindings(self, file): """ Convert EVTX to JSON using evtx_dump bindings (slower) Drop resulting JSON files in a tmp folder. """ try: filepath = Path(file) filename = filepath.name parser = PyEvtxParser(str(filepath)) with open( f"{self.tmpDir}/{str(filename)}-{self.randString()}.json", "w") as f: for record in parser.records_json(): f.write(f'{json.dumps(json.loads(record["data"]))}\n') except Exception as e: self.logger.error(f"{Fore.RED} [-] {e}")
def MatchLogFile(self, filePath): parser = PyEvtxParser(filePath) if self.logger: self.logger.info( f"Statrted processing the Event log '{filePath}'.") for record in parser.records_json(): try: data = json.loads(record["data"]) event = Event(data) self.matchAll(event) except Exception as e: if self.logger: self.logger.error(e, exc_info=True) if self.logger: self.logger.info( f"Finished processing the Event log '{filePath}'.")
def asyncGenLogs(file): EVTx = {} json_data_dir = createDir('web\json_data') print('EVTxFILE:', file) fname = filename(file) _df = pd.DataFrame() parser = PyEvtxParser(file) i = 0 # Register counter for record in parser.records_json(): data = json.loads(record['data'])['Event'] readXmlObj(data, EVTx) try: df = pd.DataFrame.from_dict(EVTx).fillna(0) _df = _df.append(df) i += 1 except Exception as e: pass EVTx = {} if i > (REG_COUNT_LIMIT - 1): break if not _df.empty: if REGEX_PATERN: try: _df = _df[_df.apply(lambda x: x.astype(str).str.contains( REGEX_PATERN, case=False, na=False)).any(axis=1)] except Exception as e: # error(e) #verbose pass _df = _df.set_index('EventRecordID').sort_values(by=['EventRecordID'], ascending=False) _df.to_json(f'{json_data_dir}\\{md5(fname)}.json', orient="table") r, c = _df.shape eel.feedback(f'Lines: {r}, Columns: {c}', md5(file)) else: pd.DataFrame(['Empty'], columns=['File' ]).to_json(f'{json_data_dir}\\{md5(fname)}.json', index=False, orient="table") eel.feedback(f'Empty', md5(file))
def load_events(log_file_name): """ Opens Sysmon logs as a readable file, and turns the events into dictionaries :param log_file_name: Sysmon evtx or xml log to be opened :return: dict of the Sysmon event log """ try: with open(log_file_name, "r", encoding='utf-8', errors='ignore') as fp: magic = fp.read(7) fp.seek(0) if magic == 'ElfFile': # log is evtx type parser = PyEvtxParser(log_file_name) dictrecords = [json.loads(rec['data']) for rec in parser.records_json()] return dictrecords, 'evtx' elif magic == '<Events': return xmltodict.parse(fp.read()), 'xml' else: raise Exception('Invalid File magic') except ExpatError: raise KeyError("Error: Format error in the Event log file")
class Evtx2es(object): def __init__(self, filepath: str) -> None: self.path = Path(filepath) self.parser = PyEvtxParser(self.path.open(mode='rb')) def gen_json(self, size: int) -> Generator: buffer: List[dict] = [] for record in self.parser.records_json(): record['data'] = json.loads(record.get('data')) eventid_field = record.get('data').get('Event').get('System').get( 'EventID') if type(eventid_field) is dict: record['data']['Event']['System'][ 'EventID'] = eventid_field.get('#text') try: status = record.get('data').get('Event').get('EventData').get( 'Status') record['data']['Event']['EventData']['Status'] = None except Exception: pass # Convert data according to ECS (sort of) # First copy system fields record['winlog'] = { 'channel': record['data']['Event']['System']['Channel'], 'computer_name': record['data']['Event']['System']['Computer'], 'event_id': record['data']['Event']['System']['EventID'], 'opcode': record['data']['Event']['System'].get('Opcode'), 'provider_guid': record['data']['Event']['System']['Provider'] ['#attributes'].get('Guid'), 'provider_name': record['data']['Event']['System']['Provider']['#attributes'] ['Name'], 'record_id': record['data']['Event']['System']['EventRecordID'], 'task': record['data']['Event']['System']['Task'], 'version': record['data']['Event']['System'].get('Version'), } try: record['winlog']['process'] = { 'pid': record['data']['Event']['System']['Execution'] ['#attributes']['ProcessID'], 'thread_id': record['data']['Event']['System']['Execution'] ['#attributes']['ThreadID'], } except KeyError: pass record.update({ 'log': { 'file': { 'name': str(self.path) } }, 'event': { 'code': record['winlog']['event_id'], 'created': record['data']['Event']['System']['TimeCreated'] ['#attributes']['SystemTime'], } }) record['@timestamp'] = record['event']['created'] # Move event attributes to ECS location record['winlog']['event_data'] = record['data']['Event'].get( 'EventData', dict()) del record['data'] if record['winlog']['event_data'] is None or len( record['winlog'] ['event_data']) == 0: # remove event_data fields if empty del record['winlog']['event_data'] else: for k, v in record['winlog']['event_data'].items(): # Normalize some known problematic fields with values switching between integers and strings with hexadecimal notation to integers if k in ('ProcessId') and type(v) == str: if v.startswith("0x"): record['winlog']['event_data'][k] = int(v, 16) else: try: record['winlog']['event_data'][k] = int(v) except ValueError: record['winlog']['event_data'][k] = 0 # Maximum limit of numeric values in Elasticsearch if type(v) is int: if v < -2**63: record['winlog']['event_data'][k] = -2**63 elif v > 2**63 - 1: record['winlog']['event_data'][k] = 2**63 - 1 buffer.append(record) if len(buffer) >= size: yield buffer buffer.clear() else: yield buffer
def rdpevtx(infile, outdir): outfilenametmp = os.path.basename(infile) + ".csv" drive, outfiledir = os.path.splitdrive(os.path.dirname(infile)) #Make Dir harddir = outdir + os.sep + outfiledir outfilename = harddir + os.sep + outfilenametmp print(f'Out Info: {outfilename}') os.makedirs(harddir, exist_ok=True) header = "'EventID'|'ComputerName'|'TimeCreated'|'Security SID'|'UserID'|'Address'|'SessionID/Logon Type/Reason/IP:Port'|'Event'|'Channel (EVTX Log)'|'SRC File'\n" #create file outfile = open(outfilename, 'w') outfile.write(header) parser = PyEvtxParser(infile) print(f'Working on: {infile}') for record in parser.records_json(): subrec = json.loads(record["data"]) lineout = "" eventid = subrec['Event']['System']['EventID'] channel = subrec['Event']['System']['Channel'] computer = subrec['Event']['System']['Computer'] systemtime = subrec['Event']['System']['TimeCreated']['#attributes'][ 'SystemTime'] #print(subrec['Event']['UserData']['EventXML'].keys()) #printjson(subrec) if eventid == 21 or eventid == 22 or eventid == 23 or eventid == 24 or eventid == 25: if channel == "Microsoft-Windows-TerminalServices-LocalSessionManager/Operational": #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'|" #1 lineout = lineout + "'" + computer + "'|" #2 lineout = lineout + "'" + systemtime + "'|" #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" #4 lineout = lineout + "'" + subrec['Event']['UserData'][ 'EventXML']['User'] + "'|" #5 #Shitty way to get the value of Address from JSON but could not get it pulled any other way #for x in retrieve_nested_value(subrec, "Address"): # lineout = lineout + "'From: " + x + "'|" if 'Address' in subrec['Event']['UserData']['EventXML'].keys(): address = ( subrec['Event']['UserData']['EventXML']['Address']) else: address = '' #or empty string lineout = lineout + "'From: " + address + "'|" #6 lineout = lineout + "'" + str( subrec['Event']['UserData']['EventXML'] ['SessionID']) + "'|" #7 lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8 lineout = lineout + "'" + channel + "'|" #9 lineout = lineout + "'" + infile + "'" #10 lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 39: if channel == "Microsoft-Windows-TerminalServices-LocalSessionManager/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" #1 lineout = lineout + "'" + computer + "'|" #2 lineout = lineout + "'" + systemtime + "'|" #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" #4 lineout = lineout + "|" #5 lineout = lineout + "|" #6 #lineout = lineout + "'" + "Session: " + str(subrec['Event']['UserData']['EventXML']['Session']) #lineout = lineout + " Reason: " + rdpdisconnect[str(subrec['Event']['UserData']['EventXML']['Reason'])] + "'|" lineout = lineout + "'" + "Session: " + str( subrec['Event']['UserData']['EventXML'] ['TargetSession']) #7 lineout = lineout + " Source: " + str( subrec['Event']['UserData']['EventXML'] ['Source']) + "'|" #7 lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8 lineout = lineout + "'" + channel + "'|" #9 lineout = lineout + "'" + infile + "'" #10 lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 40: if channel == "Microsoft-Windows-TerminalServices-LocalSessionManager/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" #1 lineout = lineout + "'" + computer + "'|" #2 lineout = lineout + "'" + systemtime + "'|" #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" #4 lineout = lineout + "|" #5 lineout = lineout + "|" #6 lineout = lineout + "'" + "Session: " + str( subrec['Event']['UserData']['EventXML']['Session']) #7 lineout = lineout + " Reason: " + rdpdisconnect[str( subrec['Event']['UserData']['EventXML'] ['Reason'])] + "'|" #7 lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8 lineout = lineout + "'" + channel + "'|" #9 lineout = lineout + "'" + infile + "'" #10 lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) #-------------------------------------------------------------------------------------------------------------------------------- if eventid == 261: if channel == "Microsoft-Windows-TerminalServices-RemoteConnectionManager/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 1149: if channel == "Microsoft-Windows-TerminalServices-RemoteConnectionManager/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "'" + subrec['Event']['UserData'][ 'EventXML']['Param2'] + "\\" + subrec['Event']['UserData'][ 'EventXML']['Param1'] + "'|" lineout = lineout + "'" + "Src IP: " + subrec['Event'][ 'UserData']['EventXML']['Param3'] + "'|" lineout = lineout + "|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) #--------------------------------------------------------------------------------------------------------------------------------- if eventid == 131 or eventid == 140: if channel == "Microsoft-Windows-RemoteDesktopServices-RdpCoreTS/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "'From: " + subrec['Event']['EventData'][ 'ClientIP'] + "'|" lineout = lineout + "|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) #--------------------------------------------------------------------------------------------------------------------------------- if eventid == 4624: if channel == "Security": #printjson(subrec) lineout = "'" + str(eventid) + "'|" #1 lineout = lineout + "'" + computer + "'|" #2 lineout = lineout + "'" + systemtime + "'|" #3 lineout = lineout + "|" #Security UserID #4 # 6 TargetDomainname # 5 TargetUserName # 4 TargetUserSid # 8 Logon Type # 11 Workstation Name # 18 IPaddress # 19 Port lineout = lineout + "'" + subrec['Event']['EventData'][ 'TargetDomainName'] + "\\" + subrec['Event']['EventData'][ 'TargetUserName'] lineout = lineout + " (" + subrec['Event']['EventData'][ 'TargetUserSid'] + ") " + "'|" #5 lineout = lineout + "'" + subrec['Event']['EventData'][ 'IpAddress'] + ":" + subrec['Event']['EventData'][ 'IpPort'] + " -> " lineout = lineout + subrec['Event']['EventData'][ 'WorkstationName'] + "'|" #6 lineout = lineout + "'Logon Type: " + str( subrec['Event']['EventData']['LogonType']) + "'|" #7 lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8 lineout = lineout + "'" + channel + "'|" #9 lineout = lineout + "'" + infile + "'" #10 lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 4625: if channel == "Security": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "|" #Security UserID # 6 TargetDomainname # 5 TargetUserName # 4 TargetUserSid # 8 FailureReason # 10 LogonType # 13 WorkstationName # 19 IPaddress # 20 Port lineout = lineout + "'" + subrec['Event']['EventData'][ 'TargetDomainName'] + "\\" + subrec['Event']['EventData'][ 'TargetUserName'] lineout = lineout + " (" + subrec['Event']['EventData'][ 'TargetUserSid'] + ") " + "'|" lineout = lineout + "'From: " + subrec['Event']['EventData'][ 'WorkstationName'] + " (" + subrec['Event']['EventData'][ 'IpAddress'] + ":" + subrec['Event']['EventData'][ 'IpPort'] + ")'|" lineout = lineout + "'Logon Type: " + str( subrec['Event']['EventData'] ['LogonType']) + " Failure Reason: " + subrec['Event'][ 'EventData']['FailureReason'] + "'|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 4634: if channel == "Security": #printjson(subrec) lineout = "'" + str(eventid) + "'|" #1 lineout = lineout + "'" + computer + "'|" #2 lineout = lineout + "'" + systemtime + "'|" #3 lineout = lineout + "|" #Security UserID #4 # 3 TargetDomainname # 1 TargetUserName # 0 TargetUserSid # 4 LogonType lineout = lineout + "'" + subrec['Event']['EventData'][ 'TargetDomainName'] + "\\" + subrec['Event']['EventData'][ 'TargetUserName'] lineout = lineout + " (" + subrec['Event']['EventData'][ 'TargetUserSid'] + ") " + "'|" #5 lineout = lineout + "|" #6 lineout = lineout + "'Logon Type: " + str( subrec['Event']['EventData']['LogonType']) + "'|" #7 lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8 lineout = lineout + "'" + channel + "'|" #9 lineout = lineout + "'" + infile + "'" #10 lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 4647: if channel == "Security": #printjson(subrec) lineout = "'" + str(eventid) + "'|" #EventID lineout = lineout + "'" + computer + "'|" #ComputerName lineout = lineout + "'" + systemtime + "'|" #Time Created lineout = lineout + "|" #Security UserID #4 # 3 TargetDomainname # 1 TargetUserName # 0 TargetUserSid # 4 LogonType lineout = lineout + "'" + subrec['Event']['EventData'][ 'TargetDomainName'] + "\\" + subrec['Event']['EventData'][ 'TargetUserName'] lineout = lineout + " (" + subrec['Event']['EventData'][ 'TargetUserSid'] + ") " + "'|" #UserID lineout = lineout + "|" #Address lineout = lineout + "|" #SessionID/Logon Type/Reason/IP:Port lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8 lineout = lineout + "'" + channel + "'|" #9 lineout = lineout + "'" + infile + "'" #10 lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 4648: if channel == "Security": if search('winlogon.exe$', subrec['Event']['EventData']['ProcessName']): #printjson(subrec) lineout = "'" + str(eventid) + "'|" #EventID lineout = lineout + "'" + computer + "'|" #ComputerName lineout = lineout + "'" + systemtime + "'|" #Time Created lineout = lineout + "|" #Security UserID # 6 TargetDomainname # 5 TargetUserName # 8 TargetServerName # 11 Processname # 12 IPaddress # 13 Port lineout = lineout + "'Subject: " + subrec['Event'][ 'EventData']['SubjectDomainName'] + "\\" + subrec[ 'Event']['EventData']['SubjectUserName'] lineout = lineout + " (LogonID: " + subrec['Event'][ 'EventData']['SubjectUserSid'] + ") " lineout = lineout + "Target: " + subrec['Event'][ 'EventData']['TargetDomainName'] + "\\" + subrec[ 'Event']['EventData'][ 'TargetUserName'] + "'|" #UserID lineout = lineout + "|'" + subrec['Event']['EventData'][ 'IpAddress'] + ":" + subrec['Event']['EventData'][ 'IpPort'] lineout = lineout + " Target: " + subrec['Event'][ 'EventData']['TargetServerName'] + " (" + subrec[ 'Event']['EventData'][ 'TargetInfo'] + ") " + "'|" #Address lineout = lineout + "|'Process: " + subrec['Event'][ 'EventData']['ProcessName'] + " (" + subrec['Event'][ 'EventData'][ 'ProcessId'] + ")'|" #SessionID/Logon Type/Reason/IP:Port lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 4778 or eventid == 4779: if channel == "Security": #printjson(subrec) lineout = "'" + str(eventid) + "'|" #EventID lineout = lineout + "'" + computer + "'|" #ComputerName lineout = lineout + "'" + systemtime + "'|" #Time Created lineout = lineout + "|" #Security UserID # 3 TargetDomainname # 1 TargetUserName # 0 TargetUserSid lineout = lineout + "'" + subrec['Event']['EventData'][ 'AccountDomain'] + "\\" + subrec['Event']['EventData'][ 'AccountName'] lineout = lineout + " (LogonID: " + subrec['Event'][ 'EventData']['LogonID'] + ") " + "'|" #UserID lineout = lineout + "|'" + subrec['Event']['EventData'][ 'ClientName'] + " (" + subrec['Event']['EventData'][ 'ClientAddress'] + ") " + "'|" #Address lineout = lineout + "|'" + subrec['Event']['EventData'][ 'SessionName'] + "'|" #SessionID/Logon Type/Reason/IP:Port lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) #--------------------------------------------------------------------------------------------------------------------------------- if eventid == 1024: if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "'Connect to: " + subrec['Event'][ 'EventData']['Value'] + "'|" lineout = lineout + "|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 1025: if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 1026: if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + subrec['Event']['EventData'][ 'CustomLevel'] + ": " + subrec['Event']['EventData'][ 'Name'] + " " + str( subrec['Event']['EventData']['Value']) + "'|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) if eventid == 1029: if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "Userid Base64 Hash: " + subrec['Event'][ 'EventData']['TraceMessage'] + "'|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) #--------------------------------------------------------------------------------------------------------------------------------- if eventid == 56: if channel == "System": printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) #System 9009 Eventid #if '#text' in subrec['Event']['System']['EventID'].keys(): # eventidsys = subrec['Event']['System']['EventID']['#text'] #else: # eventidsys = 0 #or empty string #if channel == "System": #print(f'{subrec['Event']['System'].keys()}') # print(subrec['Event']['UserData']['EventXML'].keys()) #if eventidsys == "9009": # if channel == "System": # if subrec['Event']['System']['Provider']['attributes']['Name'] == 'Desktop Window Manager': # printjson(subrec) # lineout = "'" + str(eventid) + "'|" # lineout = lineout + "'" + computer + "'|" # lineout = lineout + "'" + systemtime + "'|" # lineout = lineout + "'" + subrec['Event']['System']['Security']['#attributes']['UserID'] + "'|" # lineout = lineout + "|" # lineout = lineout + "|" # lineout = lineout + "|" # lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" # lineout = lineout + "'" + channel + "'|" # lineout = lineout + "'" + infile + "'" # lineout = lineout + "|python" # lineout = lineout + "\n" #print(f'{lineout}') # outfile.write(lineout) #--------------------------------------------------------------------------------------------------------------------------------- if eventid == 7001 or eventid == 7002: if channel == "Microsoft-Windows-Winlogon/Operational": #printjson(subrec) lineout = "'" + str(eventid) + "'|" lineout = lineout + "'" + computer + "'|" lineout = lineout + "'" + systemtime + "'|" lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes']['UserID'] + "'|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "|" lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" lineout = lineout + "'" + channel + "'|" lineout = lineout + "'" + infile + "'" lineout = lineout + "|python" lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) #print(subrec) #print(subrec['Event']['UserData']['EventXML']) #print(subrec['Event']['UserData']['EventXML']['Address']) #print(f'------------------------------------------') print("end") outfile.close()
<head> <meta charset="utf-8" /> <title> Report </title> <meta name="viewport" content="width=device-width, initial-scale=1"> <link rel="stylesheet" type="text/css" media="screen" href="style.css" /> </head> <body> <div class="wrapper"> <div class="header"> <H1> Report </H1> </div>''' html_code += f'\n<table align="center"> \n <caption><h2><b>Partition%4Diagnostic.evtx ANALYSIS REPORT for device with S/N: {serial}</b></h2></caption>' html_code += '\n<tr style="background-color:DarkGrey"> \n <th>EventRecordID</th> \n <th>connected Timestamp (UTC)</th> \n <th>Manufacturer</th> \n <th>Model</th> \n <th>Volume 1 Serial Number</th> \n <th>Volume 2 Serial Number</th> \n <th>Volume 3 Serial Number</th> \n <th>Volume 4 Serial Number</th> \n <th>Flag</th>' for record in parser.records_json(): data = json.loads(record['data']) records_dict[(data['Event']['System']['EventRecordID'])] = data #add gia na sortaro meta me event id kai na exo xronologika sosti seira if data['Event']['System']['EventID'] != 1006: IsPartitionDiagnosticEVTX = False if IsPartitionDiagnosticEVTX: #parsarw giati einai to PartitionDiagnostic evtx afou ola ta EventID einai 1006 print('Initializing parsing') print('.................') LogStartTime = records_dict[1]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC') LogEndTime = records_dict[len(records_dict)]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC') print('Parsing complete') print('.................') print('Initializing analysis') print('.................') for i in sorted(records_dict.keys()):
def ProcessPrivateRules(self, logspath): if self.logger: self.logger.info( f"Starting processing private rules on the log/s in '{logspath}' ..." ) for pubrule in self.PublicRulesContainsPrivateRules: triggered = None privRules = [] privRulesChannels = [] TriggeredEvents = {} for privrulename in pubrule.include.get("rule"): for rule in self.ruleSet: if rule.name == privrulename: privRules.append(rule) privRulesChannels.append(rule.channel.lower()) for filePath in self.LogsToProcess: parser = PyEvtxParser(filePath) for record in parser.records_json(): try: data = json.loads(record["data"]) event = Event(data) if event.Channel.lower() in privRulesChannels: for prirule in privRules: if self.match(prirule, event): if triggered == None: triggered = True triggered = triggered and True if not TriggeredEvents.get(prirule): TriggeredEvents[prirule] = [event] else: TriggeredEvents[prirule].append(event) else: break except (OSError, KeyError) as e: if self.logger: self.logger.error(e, exc_info=True) continue except Exception as e: if self.logger: self.logger.error(e, exc_info=True) if len(TriggeredEvents) != len(privRules): return False TriggeredEventsList = [] for key, val in TriggeredEvents.items(): TriggeredEventsList.append(val) TriggeredEventsWithinTheSpecifiedTime = self.ProcessTimeBetweenLogs( list(itertools.product(*TriggeredEventsList)), int(pubrule.include.get("if").get("within"))) if TriggeredEventsWithinTheSpecifiedTime: for EventSet in TriggeredEventsWithinTheSpecifiedTime: recordIDs = [] triggeredEventsData = {} privRuleNames = pubrule.include.get("rule") privateRules = [] for r in self.ruleSet: if r.name in privRuleNames: privateRules.append(r) for r in privateRules: for e in EventSet: if r.channel == e.Channel: if r.returns: fields = {} for field in r.returns: fields.update( {field: e.EventData.get(field)}) triggeredEventsData[r.name] = fields else: triggeredEventsData[r.name] = e.RawRecord for event in EventSet: recordIDs.append(event.EventRecordID) data = [ event.TimeCreatedSystemTime, recordIDs, pubrule.name, pubrule.score, pubrule.description, pubrule.reference, [], triggeredEventsData ] self.Queue.put( Alert(event, pubrule, [], privateRule=True, record=data)) else: return False
def fullparse(): filename = values['-IN-'] FullParseRecordsDict = {} AllPluggedInSerials = [] EachPluggedDeviceDict = {} IsPartitionDiagnosticEVTXFullParse = True FullParseHTMLWritten = True try: #checkarw an einai legit evtx log parser = PyEvtxParser(filename) for record in parser.records_json(): data = json.loads(record['data']) FullParseRecordsDict[(data['Event']['System']['EventRecordID'])] = data #add gia na sortaro meta me event id kai na exo xronologika sosti seira if data['Event']['System']['EventID'] != 1006: IsPartitionDiagnosticEVTXFullParse = False FullParseLogStartTime = FullParseRecordsDict[1]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC') FullParseLogEndTime = FullParseRecordsDict[len(FullParseRecordsDict)]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC') if IsPartitionDiagnosticEVTXFullParse: for i in sorted(FullParseRecordsDict.keys()): if FullParseRecordsDict[i]['Event']['EventData']['SerialNumber'] not in AllPluggedInSerials: # prwth fora pou vrskv to usb sto log opote to grafw sigoura AllPluggedInSerials.append(FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']) # pros8etw sth lista to S/N tou usb if FullParseRecordsDict[i]['Event']['EventData']['PartitionStyle'] == 1: #an einai GPT schemed media EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']] = [FullParseRecordsDict[i]['Event']['EventData']['Manufacturer'], FullParseRecordsDict[i]['Event']['EventData']['Model'], FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), ['GPT - NO VSN INFO']] # EachPluggedDeviceDict = {'serial':[manufacturer, model, first pluggedIn, lastpluggedIn, [VSN1, VSN2 klp]] else: SN0 = volumeSNParser(FullParseRecordsDict[i], 'Vbr0', FullParseRecordsDict[i]['Event']['EventData']['Vbr0'][6:18]) if FullParseRecordsDict[i]['Event']['EventData']['Vbr1'] != '': SN1 = volumeSNParser(FullParseRecordsDict[i], 'Vbr1', FullParseRecordsDict[i]['Event']['EventData']['Vbr1'][6:18]) else: SN1 = '-' if FullParseRecordsDict[i]['Event']['EventData']['Vbr2'] != '': SN2 = volumeSNParser(FullParseRecordsDict[i], 'Vbr2', FullParseRecordsDict[i]['Event']['EventData']['Vbr2'][6:18]) else: SN2 = '-' if FullParseRecordsDict[i]['Event']['EventData']['Vbr3'] != '': SN3 = volumeSNParser(FullParseRecordsDict[i], 'Vbr3', FullParseRecordsDict[i]['Event']['EventData']['Vbr3'][6:18]) else: SN3 = '-' EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']] = [FullParseRecordsDict[i]['Event']['EventData']['Manufacturer'], FullParseRecordsDict[i]['Event']['EventData']['Model'], FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), [SN0, SN1, SN2, SN3]] else: #to exw ksanavrei to S/N tou usb opote einai hdh sto dict kai apla 8elw na kanvw update to timestamp kai ta VSN if FullParseRecordsDict[i]['Event']['EventData']['PartitionStyle'] == 1: #an einai GPT schemed media opote sigoua den exei allaksei VSN afou den ta deixnei EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][3] = FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC') # kanw update mono to last pluggedin time else: #einai eite GPt pou egine unplugged h' MBR if FullParseRecordsDict[i]['Event']['EventData']['Vbr0'] != '': #VBR0 oxi keno opote exw vrei MBR plugged in record SN0 = volumeSNParser(FullParseRecordsDict[i], 'Vbr0', FullParseRecordsDict[i]['Event']['EventData']['Vbr0'][6:18]) if FullParseRecordsDict[i]['Event']['EventData']['Vbr1'] != '': SN1 = volumeSNParser(FullParseRecordsDict[i], 'Vbr1', FullParseRecordsDict[i]['Event']['EventData']['Vbr1'][6:18]) else: SN1 = '-' if FullParseRecordsDict[i]['Event']['EventData']['Vbr2'] != '': SN2 = volumeSNParser(FullParseRecordsDict[i], 'Vbr2', FullParseRecordsDict[i]['Event']['EventData']['Vbr2'][6:18]) else: SN2 = '-' if FullParseRecordsDict[i]['Event']['EventData']['Vbr3'] != '': SN3 = volumeSNParser(FullParseRecordsDict[i], 'Vbr3', FullParseRecordsDict[i]['Event']['EventData']['Vbr3'][6:18]) else: SN3 = '-' EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][3] = FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC') if SN3 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN3) if SN2 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN2) if SN1 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN1) if SN0 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN0) else: continue #next iteration giati exw vrei record eite gia MBR eite GPT alla pou einai gia unplugged FullParsehtml_code ='''<!DOCTYPE html> <html> <head> <meta charset="utf-8" /> <title> Report </title> <meta name="viewport" content="width=device-width, initial-scale=1"> <link rel="stylesheet" type="text/css" media="screen" href="style.css" /> </head> <body> <div class="wrapper"> <div class="header"> <H1> Report </H1> </div>''' FullParsehtml_code += f'\n<table align="center"> \n <caption><h2><b>Full Report for all connected devices</b></h2></caption>' FullParsehtml_code += '\n<tr style="background-color:DarkGrey"> \n <th>Media S/N</th> \n <th>Manufacturer</th> \n <th>Model</th> \n <th>First connected Timestamp (UTC)</th> \n <th>Last connected Timestamp (UTC)</th> \n <th>Every VSN recovered from the log</th>' for serial in AllPluggedInSerials: FullParsehtml_code += f'\n<tr> \n <td>{serial}</td> \n<td>{EachPluggedDeviceDict[serial][0]}</td> \n <td>{EachPluggedDeviceDict[serial][1]}</td> \n<td>{EachPluggedDeviceDict[serial][2]}</td> \n<td>{EachPluggedDeviceDict[serial][3]} </td>\n <td>' for i in range(len(EachPluggedDeviceDict[serial][4])): if EachPluggedDeviceDict[serial][4][i] == '-' or EachPluggedDeviceDict[serial][4][i] == 'Unknown Volume Type': continue FullParsehtml_code += f'{EachPluggedDeviceDict[serial][4][i]} ' FullParsehtml_code += '</td>' FullParsehtml_code += f'\n</table> \n<br>\n<p align="left" style="color:white">\n <u style="font-size:20px"> Complete Log Timeline</u> <br>\n From: {FullParseLogStartTime} <br>\nTo: {FullParseLogEndTime} <br>\n</p> \n<br> <br> <br>\n<div class="push"></div> \n</div> \n <div class="footer">--Partition%4DiagnosticParser Ver. 1.3.0</div>\n</body>\n</html>' FullParsecss_code = '''table{border-collapse:collapse;} th{text-align:center;background-color:#4a4343;color=white;} table,th,td{border:1px solid #000;} tr{text-align:center;background-color:#595555; color:white;} html, body { height: 100%; margin: 0; } .wrapper { min-height: 100%; background-color: #4a4349; /* Equal to height of footer */ /* But also accounting for potential margin-bottom of last child */ margin-bottom: -50px; font-family: "Courier New", sans-serif; color=white; } .header{ background-color: dark grey; color=white; } .header h1 { text-align: center; font-family: "Courier New", sans-serif; color=red; } .push { height: 50px; background-color: #4a4349; } .footer { height: 50px; background-color: #4a4349; color=white; text-align: right; } ''' if values['-INSAVE-'] == defaultOutputPathText: #dhl tickare gia html report alla den epelekse fakelo gia save alla afise to default try: with open('Full_Report.html', 'w', encoding='utf8') as fout: fout.write(FullParsehtml_code) with open('style.css', 'w', encoding= 'utf8') as cssout: cssout.write(FullParsecss_code) except: FullParseHTMLWritten = False else: #dhl exei dwsei output save folder try: with open(f"{values['-INSAVE-']}/Full_Report.html", 'w', encoding='utf8') as fout: fout.write(FullParsehtml_code) with open(f"{values['-INSAVE-']}/style.css", 'w', encoding= 'utf8') as cssout: cssout.write(FullParsecss_code) except: FullParseHTMLWritten = False if FullParseHTMLWritten: if values['-DISKSN-'] == '': print('Initializing parsing') print('.................') print('Parsing complete') print('.................') print('Initializing analysis') print('.................') print('Analysis complete') print('.................') print('---------------------------') print('Analysis Results') print('---------------------------') print() print() print(f'Log timeline:') print(f'Start: {FullParseLogStartTime}') print(f'End: {FullParseLogEndTime}') print() print('Full analysis report for all connected devices completed') sg.PopupOK('Full report created succesfully!', title=':)', background_color='#2a363b') else: print('Full report has been created succesfully but was unable to be written to disk\nCheck write permissions for the selected output folder!') else: sg.PopupOK('Not a PartitionDiagnostic EVTX log file!', title='!!!', background_color='#2a363b') window['-IN-'].update('') window['-DISKSN-'].update('') except Exception as e: print(e) sg.PopupOK('Error parsing the chosen EVTX log file!', title='Error', background_color='#2a363b') window['-IN-'].update('') window['-DISKSN-'].update('')
def driver(case_name, file_name, Artifact_ID): conn = pymysql.connect(host='192.168.4.188', user='******', password='******', db=case_name, charset='utf8') curs = conn.cursor() sql_usb_log = "insert into Usb_Event_Log (Artifact_ID, Event_ID,System_Time, Serial_Number, Manufacturer,Usb_Name) values (%s, %s, %s, %s, %s, %s)" file = file_name parser = PyEvtxParser(file) id = 0 try: for record in parser.records_json(): id += 1 event = json.loads(record['data'])['Event'] try: Event_ID = event['System']['EventID'] except: Event_ID = None print("Event_ID Error") try: timestamp = record['timestamp'] except: timestamp = None print("Timestamp Error") try: Instance_ID = event['UserData']['UMDFHostDeviceRequest'][ 'InstanceId'] if Instance_ID.startswith('USB'): try: serial = Instance_ID.split('\\')[2] except: serial = Instance_ID try: prod = Instance_ID.split('\\')[1].split('&')[0].split( 'VID_')[1] except: prod = None try: ven = Instance_ID.split('\\')[1].split('&')[1].split( 'PID_')[1] except: ven = None try: t1 = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S.%f UTC') t1.strftime('%Y-%m-%d %H:%M:%S') except: t1 = datetime.datetime(1900, 1, 1, 00, 00, 00) elif Instance_ID.startswith('SWD'): try: serial = Instance_ID.split('#')[2].split('&')[0] except: serial = reg.search(Instance_ID)[0] try: ven = Instance_ID.split('&')[1].split('VEN_')[1] except: ven = None try: prod = Instance_ID.split('&')[2].split('PROD_')[1] except: prod = None try: t1 = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S.%f UTC') t1.strftime('%Y-%m-%d %H:%M:%S') except: t1 = datetime.datetime(1900, 1, 1, 00, 00, 00) else: continue except: print("Instance_ID Error") curs.execute(sql_usb_log, (Artifact_ID, Event_ID, t1, serial, ven, prod)) except: pass conn.commit() conn.close()
def partition(case_name, file_name, Artifact_ID): conn = pymysql.connect(host='192.168.4.188', user='******', password='******', db=case_name, charset='utf8') curs = conn.cursor() file = file_name sql_partition_log = "insert into Usb_Partition_Log(Artifact_ID,Manufacturer , Model, Serial_Number , VSN) values (%s, %s, %s, %s, %s)" parser = PyEvtxParser(file) instance = 0 id = 0 table = [] try: for record in parser.records_json(): id += 1 data = json.loads(record['data'])['Event'] try: Manufacture = data['EventData']['Manufacturer'] except: Manufacture = None print("Manufacture error") try: Model = data['EventData']['Model'] except: Model = None print("Model error") try: ParentId = data['EventData']['ParentId'] Serial_Number = ParentId.split('\\')[2] except: Serial_Number = None print("Serial Number error") try: VBR = data['EventData']['Vbr0'] if VBR[6:14] == '4e544653': vsn_reverse = VBR[144:152] #NTFS vsn = '0x' count = 9 for index in range(4): count -= 2 vsn = vsn + vsn_reverse[count - 1:count + 1] elif VBR[164:174] == '4641543332': vsn_reverse = VBR[134:142] # FAT32 vsn = '0x' count = 9 for index in range(4): count -= 2 vsn = vsn + vsn_reverse[count - 1:count + 1] elif VBR[6:16] == '4558464154': vsn_reverse = VBR[200:208] # exfat vsn = '0x' count = 9 for index in range(4): count -= 2 vsn = vsn + vsn_reverse[count - 1:count + 1] elif VBR[108:116] == '46415431': vsn_reverse = VBR[78:86] # fat16 vsn = '0x' count = 9 for index in range(4): count -= 2 vsn = vsn + vsn_reverse[count - 1:count + 1] else: vsn = None except: vsn = None print("vsn error") if Manufacture != None and Model != None and Serial_Number != None and vsn != None: table.append([Manufacture, Model, Serial_Number, vsn]) new_table = [] for element in table: if element not in new_table: new_table.append(element) for column in new_table: curs.execute( sql_partition_log, (Artifact_ID, column[0], column[1], column[2], column[3])) except: print("sql error") conn.commit() conn.close()
def nom_file(filename, welm_map): parser = PyEvtxParser(filename) # Open Records for record in parser.records_json(): data = json.loads(record['data']) try: # Event Log event event = {'recordid': str(record['event_record_id'])} event.update(get_section(data['Event']['System'])) if data['Event'].get('EventData'): event['event_data'] = get_section(data['Event']['EventData']) if data['Event'].get('UserData'): #print(data['Event'].get('UserData')) if data['Event']['UserData'].get('EventXML'): event['event_data'] = get_section( data['Event']['UserData']['EventXML']) else: # not sure about what other namesspaces are here so for now just this loop for ns in data['Event']['UserData']: event['event_data'] = get_section( data['Event']['UserData'][ns]) if isinstance(event['eventid'], dict): print(event['eventid']) print("#" * 20) print(json.dumps(event, indent=3)) print("#" * 20) print(json.dumps(data, indent=3)) key = make_key( event.get('channel') or '', event['provider']['name'], event['eventid']) if key in welm_map: if welm_map[key][ 'swap_mode'] and welm_map[key]['params'] != []: if event.get('event_data') or False: swap_target = 'event_data' elif event.get('user_data') or False: swap_target = 'user_data' else: swap_target = None event['message'] = welm_map[key]['format_string'] if swap_target: swap_values = ['bump'] for param in welm_map[key]['params']: swap_values.append(event[swap_target].get(param) or "") #print(key) #print(welm_map[key]['format_string']) #print(welm_map[key]['params']) #print(swap_values) try: event['message'] = welm_map[key][ 'format_string'].format(*swap_values) except: event['message'] = welm_map[key]['format_string'] else: event['message'] = welm_map[key]['format_string'] else: event[ 'message'] = "{} | {} | {} | Unknown Message String".format( event['eventid'], event.get('channel') or '', event['provider']['name']) # Raw Document event['raw'] = record['data'] yield event except KeyError as errormsg: print("Soemthing went wrong parsing this event") print(json.dumps(data, indent=4))
def parser_simple(path, opath=None): import time cursor = sql_connection() sql_initialitation(cursor) parser = PyEvtxParser(path) connections_key = 100000000 thread_key = 0 files_inserted = [] full_process_inserted = [] pipes_inserted = [] threads_inserted = [] start = time.time() for record in parser.records_json(): event = json.loads(record['data']) # Process Creation if event["Event"]["System"]["EventID"] == 1: try: if event["Event"]["EventData"]["ProcessGuid"] not in full_process_inserted: #print("Full inserted") query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image","IntegrityLevel",' \ '"TerminalSessionId", "User")' \ " VALUES ('{}','{}','{}','{}','{}', '{}')" \ ' ON CONFLICT ("ProcessGuid") DO UPDATE SET "IntegrityLevel" = ' \ 'EXCLUDED."IntegrityLevel", "TerminalSessionId"' \ ' = EXCLUDED."TerminalSessionId", "User" = EXCLUDED."User";'.format( (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["ProcessId"], event["Event"]["EventData"]["Image"], event["Event"]["EventData"]["IntegrityLevel"], event["Event"]["EventData"]["TerminalSessionId"], event["Event"]["EventData"]["User"]) cursor.execute(query_process) full_process_inserted.append(event["Event"]["EventData"]["ProcessGuid"]) except Exception as e: logger.error("Error query_process 1: " + str(e) + " Event: " + str(event["Event"])) try: if event["Event"]["EventData"]["Image"] not in files_inserted: if "OriginalFileName" in event["Event"]["EventData"]: original = ",'" + str(event["Event"]["EventData"]["OriginalFileName"]) + "'" key = ',"OriginalFileName"' else: original = "" key = "" except Exception as e: logger.error("Error query_file 1: " + str(e) + " Event: " + str(event["Event"])) try: query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["ParentImage"]).lower(), event["Event"]["EventData"]["ParentProcessId"], event["Event"]["EventData"]["ParentImage"]) cursor.execute(query_pprocess) except Exception as e: logger.error("Error query_pprocess_exist 1: " + str(e) + " Event: " + str(query_pprocess)) try: if event["Event"]["EventData"]["CommandLine"]: event["Event"]["EventData"]["CommandLine"] = str( event["Event"]["EventData"]["CommandLine"]).replace("'", "\"") if event["Event"]["EventData"]["CurrentDirectory"]: event["Event"]["EventData"]["CurrentDirectory"] = str( event["Event"]["EventData"]["CurrentDirectory"]).replace("'", "\"") query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \ '"ExtraInfo","ExtraInfo2")' \ " VALUES ('{}','{}','{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateProcess", (event["Event"]["EventData"]["ParentImage"]).lower(), event["Event"]["EventData"]["LogonGuid"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["CommandLine"], event["Event"]["EventData"]["CurrentDirectory"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 1: " + str(e) + " Event: " + str(query_action)) try: query_user = '******' \ " ('{}','{}','{}',{}) ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["LogonGuid"], event["Event"]["EventData"]["User"], event["Event"]["EventData"]["LogonId"], event["Event"]["EventData"]["TerminalSessionId"]) cursor.execute(query_user) except Exception as e: logger.error("Error 1: " + str(e) + " Event: " + str(query_user)) # File creation time changed if event["Event"]["System"]["EventID"] == 2: logger.info("ToDo") # Network connection if event["Event"]["System"]["EventID"] == 3: try: connections_key = str(event["Event"]["EventData"]["SourceIp"])+str(event["Event"]["EventData"]["DestinationIp"]) query_connection = 'INSERT INTO public."Connections" ("ConnectionId","Protocol","SourceIp","SourceHostname",' \ '"SourcePort","DestinationIsIpv6","DestinationIp","DestinationHostname","DestinationPort") ' \ "VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format( connections_key, event["Event"]["EventData"]["Protocol"], event["Event"]["EventData"]["SourceIp"], event["Event"]["EventData"]["SourceHostname"], event["Event"]["EventData"]["SourcePort"], event["Event"]["EventData"]["DestinationIsIpv6"], event["Event"]["EventData"]["DestinationIp"], event["Event"]["EventData"]["DestinationHostname"], event["Event"]["EventData"]["DestinationPort"]) cursor.execute(query_connection) except Exception as e: logger.error("Error query_connection 3: " + str(e) + " Event: " + str(query_connection)) try: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \ '"ExtraInfo")' \ " VALUES ('{}','{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateConnection", (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["User"], connections_key, event["Event"]["EventData"]["Initiated"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 3: " + str(e) + " Event: " + str(query_action)) try: insert_process(cursor, event) except Exception as e: logger.error("Error query_pprocess_exist 3: " + str(e) + " Event: " + str(event["Event"])) # Process Terminated if event["Event"]["System"]["EventID"] == 5: try: # Destination Process query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["ProcessId"], event["Event"]["EventData"]["Image"]) cursor.execute(query_process) except Exception as e: logger.error("Error query_sprocess 5: " + str(e) + " Event: " + str(query_process)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "ProcessTerminated", (event["Event"]["EventData"]["Image"]).lower(), (event["Event"]["EventData"]["Image"]).lower()) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 5: " + str(e) + " Event: " + str(query_action)) # Kernel driver loaded if event["Event"]["System"]["EventID"] == 6: logger.info("ToDo") # Image loaded if event["Event"]["System"]["EventID"] == 7: try: # Process insert_process(cursor, event) except Exception as e: logger.error("Error query_sprocess 7: " + str(e) + " Event: " + str(query_sprocess)) try: # File if "Description" not in event["Event"]["EventData"]: event["Event"]["EventData"]["Description"] = "" if "OriginalFileName" not in event["Event"]["EventData"]: event["Event"]["EventData"]["OriginalFileName"] = "" query_file = 'INSERT INTO public."Files" ("Filename","FileVersion","Description","Product","Company","OriginalFileName","Hashes","Signed","Signature","SignatureStatus") ' \ " VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT" \ ' ("Filename") DO UPDATE SET' \ ' "FileVersion" = EXCLUDED."FileVersion", "Description" = EXCLUDED."Description",' \ ' "Product" = EXCLUDED."Product", "Company" = EXCLUDED."Company",' \ '"OriginalFileName" = EXCLUDED."OriginalFileName","Hashes" = EXCLUDED."Hashes"' \ ',"Signed" = EXCLUDED."Signed","Signature" = EXCLUDED."Signature",' \ '"SignatureStatus" = EXCLUDED."SignatureStatus";'.format( "f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(), event["Event"]["EventData"]["FileVersion"], event["Event"]["EventData"]["Description"], event["Event"]["EventData"]["Product"], event["Event"]["EventData"]["Company"], event["Event"]["EventData"]["OriginalFileName"], event["Event"]["EventData"]["Hashes"], event["Event"]["EventData"]["Signed"], event["Event"]["EventData"]["Signature"], event["Event"]["EventData"]["SignatureStatus"]) cursor.execute(query_file) except Exception as e: logger.error("Error query_file 7: " + str(e) + " Event: " + str(query_file)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "LoadImage", "f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(), (event["Event"]["EventData"]["Image"]).lower()) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 7: " + str(e) + " Event: " + str(query_action)) # Create Remote Thread if event["Event"]["System"]["EventID"] == 8: try: # Source Process query_sprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image")' \ " VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;"\ .format((event["Event"]["EventData"]["SourceImage"]).lower(), event["Event"]["EventData"]["SourceProcessId"], (event["Event"]["EventData"]["SourceImage"]).lower()) cursor.execute(query_sprocess) except Exception as e: logger.error("Error query_sprocess 8: " + str(e) + " Event: " + str(query_sprocess)) try: # Target Process query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["TargetProcessId"], (event["Event"]["EventData"]["TargetImage"]).lower()) cursor.execute(query_tprocess) except Exception as e: logger.error("Error query_tprocess 8: " + str(e) + " Event: " + str(query_tprocess)) try: # Thread thread_key = str(event["Event"]["EventData"]["TargetProcessGuid"]) + ":" + str(event["Event"]["EventData"]["NewThreadId"]) query_thread = 'INSERT INTO public."Threads" ("ThreadId","ThreadNId","ProcessGuid","StartAddress",' \ '"StartModule", "StartFunction")' \ " VALUES ('{}','{}','{}','{}','{}','{}') ON CONFLICT " \ '("ThreadId") DO UPDATE SET "StartAddress" = ' \ 'EXCLUDED."StartAddress", "StartModule"' \ ' = EXCLUDED."StartModule", "StartFunction"' \ ' = EXCLUDED."StartFunction";'.format( thread_key, event["Event"]["EventData"]["NewThreadId"], (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["StartAddress"], event["Event"]["EventData"]["StartModule"], event["Event"]["EventData"]["StartFunction"]) cursor.execute(query_thread) except Exception as e: logger.error("Error query_thread 8: " + str(e) + " Event: " + str(query_thread)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateRemoteThread", (event["Event"]["EventData"]["SourceImage"]).lower(), thread_key) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 8: " + str(e) + " Event: " + str(query_action)) # Raw access read if event["Event"]["System"]["EventID"] == 9: logger.info("ToDo") # Process Access if event["Event"]["System"]["EventID"] == 10: try: query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["SourceImage"]).lower(), event["Event"]["EventData"]["SourceProcessId"], (event["Event"]["EventData"]["SourceImage"]).lower()) cursor.execute(query_pprocess) except Exception as e: logger.error("Error query_Pprocess 10: " + str(e) + " Event: " + str(query_pprocess)) try: query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \ "VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format( (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["TargetProcessId"], (event["Event"]["EventData"]["TargetImage"]).lower()) cursor.execute(query_tprocess) except Exception as e: logger.error("Error query_tprocess 10: " + str(e) + " Event: " + str(query_tprocess)) try: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId",' \ '"ExtraInfo","ExtraInfo2")' \ " VALUES ('{}','{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "ProcessAccess", (event["Event"]["EventData"]["SourceImage"]).lower(), (event["Event"]["EventData"]["TargetImage"]).lower(), event["Event"]["EventData"]["GrantedAccess"], event["Event"]["EventData"]["CallTrace"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 10: " + str(e) + " Event: " + str(query_action)) # File create if event["Event"]["System"]["EventID"] == 11: # Create File try: # Process insert_process(cursor, event) except Exception as e: print("Error query_sprocess 11: " + str(e) + " Event: " + str(event["Event"])) try: # File query_file = 'INSERT INTO public."Files" ("Filename","CreationUtcTime") ' \ "VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format( "f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower(), event["Event"]["EventData"]["CreationUtcTime"]) cursor.execute(query_file) except Exception as e: logger.error("Error query_file 11: " + str(e) + " Event: " + str(query_file)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "CreateFile", (event["Event"]["EventData"]["Image"]).lower(), "f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower()) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 11: " + str(e) + " Event: " + str(query_action)) # Registry Key Operation if event["Event"]["System"]["EventID"] == 12 or event["Event"]["System"]["EventID"] == 13 \ or event["Event"]["System"]["EventID"] == 14: if event["Event"]["EventData"]["TargetObject"]: event["Event"]["EventData"]["TargetObject"] = str( event["Event"]["EventData"]["TargetObject"]).replace("'", "\"") try: # Process insert_process(cursor, event) except Exception as e: logger.error("Error query_process 12-13-14: " + str(e) + " Event: " + str(event["Event"])) try: # RegistryKey if event["Event"]["System"]["EventID"] == 13: if event["Event"]["EventData"]["Details"]: event["Event"]["EventData"]["Details"] = str( event["Event"]["EventData"]["Details"]).replace("'", "\"") query_key = 'INSERT INTO public."RegistryKeys" ("Key","Details") ' \ "VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format( event["Event"]["EventData"]["TargetObject"], event["Event"]["EventData"]["Details"]) else: query_key = 'INSERT INTO public."RegistryKeys" ("Key") ' \ "VALUES ('{}') ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["TargetObject"]) cursor.execute(query_key) except Exception as e: logger.error("Error query_key 12-13-14: " + str(e) + " Event: " + str(query_key)) try: # Action if "SetValue" in event["Event"]["EventData"]["EventType"]: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId","ExtraInfo")' \ " VALUES ('{}','{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "RegistryKey-" + event["Event"]["EventData"]["EventType"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["TargetObject"], event["Event"]["EventData"]["Details"]) else: query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], "RegistryKey-" + event["Event"]["EventData"]["EventType"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["TargetObject"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 12-13-14: " + str(e) + " Event: " + str(query_action)) # File create stream hash if event["Event"]["System"]["EventID"] == 15: logger.info("ToDo") # File create stream hash if event["Event"]["System"]["EventID"] == 15: logger.info("ToDo") # Pipe event if event["Event"]["System"]["EventID"] == 17 or event["Event"]["System"]["EventID"] == 18: try: insert_process(cursor, event) except Exception as e: print("Error insert_process 17-18: " + str(e) + " Event: " + str(event["Event"])) try: # Pipe if event["Event"]["EventData"]["PipeName"] not in pipes_inserted: query_pipe = 'INSERT INTO public."Pipes" ("PipeName") ' \ "VALUES ('{}');".format( event["Event"]["EventData"]["PipeName"]) cursor.execute(query_pipe) pipes_inserted.append(event["Event"]["EventData"]["PipeName"]) except Exception as e: logger.error("Error query_file 17-18: " + str(e) + " Event: " + str(query_pipe)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}');".format( event["Event"]["EventData"]["UtcTime"], event["Event"]["EventData"]["EventType"], (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["PipeName"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 17-18: " + str(e) + " Event: " + str(query_action)) # WMI event if event["Event"]["System"]["EventID"] == 19 or event["Event"]["System"]["EventID"] == 20 \ or event["Event"]["System"]["EventID"] == 21: logger.info("ToDo") # DNS if event["Event"]["System"]["EventID"] == 22: try: insert_process(cursor, event) except Exception as e: logger.error("Error insert_process 22: " + str(e) + " Event: " + str(event["Event"])) try: # Query query_dnsquery = 'INSERT INTO public."DNSQuery" ("QueryName") ' \ "VALUES ('{}') ON CONFLICT DO NOTHING;".format( event["Event"]["EventData"]["QueryName"]) cursor.execute(query_dnsquery) except Exception as e: logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsquery)) try: # Resolution query_dnsresolution = 'INSERT INTO public."DNSResolution" ("UtcTime","QueryName","QueryStatus","QueryResults") ' \ "VALUES ('{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format( event["Event"]["EventData"]["UtcTime"], event["Event"]["EventData"]["QueryName"], event["Event"]["EventData"]["QueryStatus"], event["Event"]["EventData"]["QueryResults"]) cursor.execute(query_dnsresolution) except Exception as e: logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsresolution)) try: # Action query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \ " VALUES ('{}','{}','{}','{}') ;".format( event["Event"]["EventData"]["UtcTime"], "DnsRequest", (event["Event"]["EventData"]["Image"]).lower(), event["Event"]["EventData"]["QueryName"]) cursor.execute(query_action) except Exception as e: logger.error("Error query_action 22: " + str(e) + " Event: " + str(query_action)) cursor.close() end = time.time() logger.info("Time to process file %s seconds ---" % (end - start))
def parseevtx(infile, outdir, coldelimiter, jsonfile): outfilenametmp = os.path.basename(infile) + ".csv" if jsonfile == 1: jsonoutfilenametmp = os.path.basename(infile) + ".json" drive, outfiledir = os.path.splitdrive(os.path.dirname(infile)) #Make Dir harddir = outdir + os.sep + outfiledir outfilename = harddir + os.sep + outfilenametmp os.makedirs(harddir, exist_ok=True) print(f'CVS Out Info : {outfilename}') if jsonfile == 1: jsonoutfilename = harddir + os.sep + jsonoutfilenametmp print(f'JSON Out Info: {jsonoutfilename}') jsonoutfile = open(jsonoutfilename, 'w') # #1 #2 #3 #4 #5 #6 #7 #8 #9 #10 #11 header = "'EventID'|'ComputerName'|'TimeCreated'|'Security SID'|'UserID'|'Address'|'Script/IP:Port'|'Payload/Path/Correlation Activity ID'|'Event'|'Channel (EVTX Log)'|'SRC File'\n" #create file outfile = open(outfilename, 'w') outfile.write( "Please note that depending on the CSV deliminater used, the fields may not line up.\n" ) outfile.write(header) parser = PyEvtxParser(infile) print(f'Working on: {infile}') for record in parser.records_json(): subrec = json.loads(record["data"]) lineout = "" try: eventid = subrec['Event']['System']['EventID'] #['#text'] except: eventid = subrec['Event']['System']['EventID']['#text'] channel = subrec['Event']['System']['Channel'] computer = subrec['Event']['System']['Computer'] systemtime = subrec['Event']['System']['TimeCreated']['#attributes'][ 'SystemTime'] #print(subrec['Event']['UserData']['EventXML'].keys()) if jsonfile == 1: if channel == "Windows PowerShell" or channel == "Microsoft-Windows-WinRM/Operational" or channel == "Microsoft-Windows-PowerShell/Operational" or channel == "Security" or channel == "System": jsonoutfile.write(printjson(subrec)) jsonoutfile.write("\n") #print(f'EID: {eventid}') #-------------------------------------------------------------------------------------------------------------------------------- #Windows PowerShell.evtx if eventid == 400 or eventid == 403 or eventid == 600: if channel == "Windows PowerShell": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['Data']['#text']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- #Microsoft-Windows-WinRM/Operational.evtx if eventid == 6: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Connecting remotely to: " + subrec[ 'Event']['EventData'][ 'connection'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 81: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Operation Name: " + subrec['Event'][ 'EventData']['operationName'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 82: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Operation: " + subrec['Event'][ 'EventData'][ 'operation'] + " -- Resource URI: " + subrec[ 'Event']['EventData'][ 'resourceURI'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 134: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Operation: " + subrec['Event'][ 'EventData']['operationName'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) # Need an example log related to powershell; now it prints out all 142 events if eventid == 142: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'OperatinName: " + subrec['Event'][ 'EventData'][ 'operationName'] + " -- Error Code :'" + str( subrec['Event']['EventData'] ['errorCode']) + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 169: if channel == "Microsoft-Windows-WinRM/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "'" + subrec['Event']['EventData'][ 'username'] + "'" + coldelimiter lineout = lineout + "" + coldelimiter #6 lineout = lineout + "'Authentication Mechanism: " + subrec[ 'Event']['EventData'][ 'authenticationMechanism'] + "'" + coldelimiter #7 try: lineout = lineout + "'" + subrec['Event']['System'][ 'Correlation']['#attributes'][ 'ActivityID'] + "'" + coldelimiter #8 except: lineout = lineout + "''" + coldelimiter lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- #Microsoft-Windows-PowerShell/Operational.evtx if eventid == 4100: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['ContextInfo']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "'" + subrec['Event']['EventData'][ 'Payload'] + "'" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 4103: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) ScriptBlockText address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['ContextInfo']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "'" + subrec['Event']['EventData'][ 'Path'] + "'" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 4104: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 tmpstr = '' tmpstr = tmpstr.join( subrec['Event']['EventData']['ScriptBlockText']) lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7 lineout = lineout + "'" + subrec['Event']['EventData'][ 'Path'] + "'" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) if eventid == 40961: if channel == "Microsoft-Windows-PowerShell/Operational": try: #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- #Security if eventid == 4688: if channel == "Security": try: if re.search( 'powershell', subrec['Event']['EventData']['NewProcessName'], re.IGNORECASE): #printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['EventData'][ 'SubjectDomainName'] + "\\" + subrec['Event'][ 'EventData']['SubjectUserName'] + " (" + subrec[ 'Event']['EventData'][ 'SubjectUserSid'] + ")'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 try: tmpstr = subrec['Event']['EventData'][ 'CommandLine'] try: tmpppn = subrec['Event']['EventData'][ 'ParentProcessName'] except: tmpppn = '' if tmpstr: lineout = lineout + "'" + tmpppn + " -> " + subrec[ 'Event']['EventData'][ 'CommandLine'] + "'" + coldelimiter #7 else: lineout = lineout + "'" + tmpppn + " -> " + subrec[ 'Event']['EventData'][ 'NewProcessName'] + "'" + coldelimiter #7 except: lineout = lineout + "'" + tmpppn + " -> " + subrec[ 'Event']['EventData'][ 'NewProcessName'] + "'" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) #-------------------------------------------------------------------------------------------------------------------------------- if eventid == 7030 or eventid == 7040 or eventid == 7045: if channel == "System": try: printjson(subrec) address = '' lineout = "'" + str(eventid) + "'" + coldelimiter #1 lineout = lineout + "'" + computer + "'" + coldelimiter #2 lineout = lineout + "'" + systemtime + "'" + coldelimiter #3 lineout = lineout + "'" + subrec['Event']['System'][ 'Security']['#attributes'][ 'UserID'] + "'" + coldelimiter #4 lineout = lineout + "" + coldelimiter #5 lineout = lineout + "" + coldelimiter #6 lineout = lineout + "" + coldelimiter #7 lineout = lineout + "" + coldelimiter #8 lineout = lineout + "'" + eventidtxt[str( eventid)] + "'" + coldelimiter #9 lineout = lineout + "'" + channel + "'" + coldelimiter #10 lineout = lineout + "'" + infile + "'" #11 lineout = lineout + "\n" #print(f'{lineout}') outfile.write(lineout) except: parseproblem(subrec, outfile) print("end") outfile.close() if jsonfile == 1: jsonoutfile.close()
class Evtx2es(object): def __init__(self, filepath: str): self.path = Path(filepath).resolve() self.parser = PyEvtxParser(self.path.open(mode="rb")) def format_record(self, record: dict) -> dict: record["data"] = orjson.loads(record.get("data")) eventid_field = record.get("data", {}).get("Event", {}).get("System", {}).get("EventID") if type(eventid_field) is dict: record["data"]["Event"]["System"]["EventID"] = eventid_field.get( "#text") try: status = record.get("data").get("Event").get("EventData").get( "Status") record["data"]["Event"]["EventData"]["Status"] = None except Exception: pass # Convert data according to ECS (sort of) # First copy system fields record["winlog"] = { "channel": record["data"]["Event"]["System"]["Channel"], "computer_name": record["data"]["Event"]["System"]["Computer"], "event_id": record["data"]["Event"]["System"]["EventID"], "opcode": record["data"]["Event"]["System"].get("Opcode"), "provider_guid": record["data"]["Event"]["System"]["Provider"]["#attributes"].get( "Guid"), "provider_name": record["data"]["Event"]["System"]["Provider"]["#attributes"] ["Name"], "record_id": record["data"]["Event"]["System"]["EventRecordID"], "task": record["data"]["Event"]["System"]["Task"], "version": record["data"]["Event"]["System"].get("Version"), } try: record["winlog"]["process"] = { "pid": record["data"]["Event"]["System"]["Execution"]["#attributes"] ["ProcessID"], "thread_id": record["data"]["Event"]["System"]["Execution"]["#attributes"] ["ThreadID"], } except KeyError: pass except TypeError: pass try: record["userdata"] = { "address": record["data"]["Event"]["UserData"]["EventXML"]["Address"], "sessionid": record["data"]["Event"]["UserData"]["EventXML"]["SessionID"], "user": record["data"]["Event"]["UserData"]["EventXML"]["User"], } except KeyError: pass except TypeError: pass record.update({ "log": { "file": { "name": str(self.path) } }, "event": { "code": record["winlog"]["event_id"], "created": record["data"]["Event"]["System"]["TimeCreated"]["#attributes"] ["SystemTime"], }, }) record["@timestamp"] = record["event"]["created"] # Move event attributes to ECS location record["winlog"]["event_data"] = record["data"]["Event"].get( "EventData", dict()) del record["data"] if (record["winlog"]["event_data"] is None or len(record["winlog"]["event_data"]) == 0): # remove event_data fields if empty del record["winlog"]["event_data"] else: if record["winlog"]["event_data"]: for k, v in record["winlog"]["event_data"].items(): # Normalize some known problematic fields with values switching between integers and strings with hexadecimal notation to integers if k in ("ProcessId") and type(v) == str: if v.startswith("0x"): record["winlog"]["event_data"][k] = int(v, 16) else: try: record["winlog"]["event_data"][k] = int(v) except ValueError: record["winlog"]["event_data"][k] = 0 # Maximum limit of numeric values in Elasticsearch if type(v) is int: if v < -(2**63): record["winlog"]["event_data"][k] = -(2**63) elif v > 2**63 - 1: record["winlog"]["event_data"][k] = 2**63 - 1 return record def gen_records(self, size: int) -> Generator: """A generator that reads records from an Evtx file and generates a dict for each record. Args: size (int): Buffer size. Yields: Generator: Yields List[dict]. """ buffer: List[dict] = list() # generates records for record in self.parser.records_json(): formatted_record: dict = self.format_record(record) buffer.append(formatted_record) if len(buffer) >= size: yield buffer buffer.clear() else: yield buffer