def parse(self):
parser = PyEvtxParser(self.eventfile)
for record in parser.records_json():
rec = json.loads(record['data'])['Event']
data = {}
# Common fields
data['event.created'] = record.get(
'timestamp',
rec['System']['TimeCreated']['#attributes']['SystemTime'])
if isinstance(rec['System']['EventID'], dict):
data['event.code'] = str(rec['System']['EventID']['#text'])
else:
data['event.code'] = str(rec['System']['EventID'])
data['event.provider'] = rec['System']['Provider']['#attributes'][
'Name']
data['event.dataset'] = rec['System']['Channel']
if 'Security' in rec['System']:
try:
data['user.id'] = rec['System']['Security']['#attributes'][
'UserID']
except Exception:
pass
try:
data['process.pid'] = rec['System']['Execution'][
'#attributes']['ProcessID']
data['process.thread.id'] = rec['System']['Execution'][
'#attributes']['ThreadID']
except Exception:
pass
# Events not defined in data_json
if not data['event.code'] in self.data_json.keys(
) or data['event.provider'] != self.data_json[
data['event.code']]['provider']:
# EventData, UserData are just reproduced as dictionaries
if 'EventData' in rec:
data['EventData'] = rec['EventData']
if 'UserData' in rec:
data['UserData'] = rec['UserData']
yield data
continue
# Selected events
data['description'] = self.data_json[
data['event.code']]["description"]
for field in ['category', 'type', 'action']:
if field in self.data_json[data['event.code']]:
data['event.{}'.format(field)] = self.data_json[
data['event.code']][field]
if 'path' not in self.data_json[data['event.code']].keys():
yield data
continue
# Extra fields
for x, item in self.data_json[data['event.code']]['path'].items():
self.get_xpath_data(x, item, rec, data)
yield data
def test_it_works_with_json(small_sample):
parser = PyEvtxParser(small_sample)
records = list(parser.records_json())
assert len(records) == 7
assert records[0]['event_record_id'] == 7
assert records[0]['timestamp'].endswith('UTC')
assert json.loads(records[0]['data'])['Event']['System']['EventID'] == 4673
def parserecord(filename,
num_items,
logBufferSize,
index,
nodes,
debug,
support_queue,
token=""):
count = 0
JSONevents = ""
logBufferLength = 0
count_postedrecord = 0
parser = PyEvtxParser(filename)
for record in parser.records_json():
count = count + 1
logBufferLength = logBufferLength + 1
record = json.loads(record['data'])
if isinstance(record['Event']['System']['EventID'], int):
eventid = record['Event']['System']['EventID']
record['Event']['System']['EventID'] = {}
record['Event']['System']['EventID']['#text'] = eventid
# Same thing with their Data field
try:
if isinstance(record['Event']['EventData']['Data'], str):
tmp = record['Event']['EventData']['Data']
record['Event']['EventData']['Data'] = {}
record['Event']['EventData']['Data']['#text'] = tmp
except KeyError:
pass
except TypeError:
pass
JSONevents = JSONevents + '{"index": {}}\n'
JSONevents = JSONevents + json.dumps(record) + "\n"
# Dump log buffer when full
if logBufferLength >= int(logBufferSize):
count_postedrecord = count_postedrecord + logBufferSize
dump_batch(JSONevents, index, nodes, count_postedrecord, num_items,
debug, support_queue, token)
JSONevents = ""
logBufferLength = 0
if logBufferLength > 0:
count_postedrecord = count_postedrecord + logBufferLength
dump_batch(JSONevents, index, nodes, count_postedrecord, num_items,
debug, support_queue, token)
logBufferLength = 0
JSONevents = ""
return count_postedrecord
class Evtx2es(object):
def __init__(self, input_path: Path) -> None:
self.path = input_path
self.parser = PyEvtxParser(self.path.open(mode="rb"))
def gen_records(self, shift: Union[str, datetime], multiprocess: bool,
chunk_size: int) -> Generator:
"""Generates the formatted Eventlog records chunks.
Args:
multiprocess (bool): Flag to run multiprocessing.
chunk_size (int): Size of the chunk to be processed for each process.
Yields:
Generator: Yields List[dict].
"""
gen_path = iter(lambda: str(self.path), None)
gen_shift = iter(lambda: shift, None)
if multiprocess:
with Pool(cpu_count()) as pool:
results = pool.starmap_async(
process_by_chunk,
zip(
generate_chunks(chunk_size,
self.parser.records_json()),
gen_path,
gen_shift,
))
yield list(chain.from_iterable(results.get(timeout=None)))
else:
buffer: List[List[dict]] = list()
for records in generate_chunks(chunk_size,
self.parser.records_json()):
if chunk_size <= len(buffer):
yield list(chain.from_iterable(buffer))
buffer.clear()
else:
buffer.append(
process_by_chunk(records, gen_path, gen_shift))
else:
yield list(chain.from_iterable(buffer))
def runUsingBindings(self, file):
"""
Convert EVTX to JSON using evtx_dump bindings (slower)
Drop resulting JSON files in a tmp folder.
"""
try:
filepath = Path(file)
filename = filepath.name
parser = PyEvtxParser(str(filepath))
with open(
f"{self.tmpDir}/{str(filename)}-{self.randString()}.json",
"w") as f:
for record in parser.records_json():
f.write(f'{json.dumps(json.loads(record["data"]))}\n')
except Exception as e:
self.logger.error(f"{Fore.RED} [-] {e}")
def MatchLogFile(self, filePath):
parser = PyEvtxParser(filePath)
if self.logger:
self.logger.info(
f"Statrted processing the Event log '{filePath}'.")
for record in parser.records_json():
try:
data = json.loads(record["data"])
event = Event(data)
self.matchAll(event)
except Exception as e:
if self.logger:
self.logger.error(e, exc_info=True)
if self.logger:
self.logger.info(
f"Finished processing the Event log '{filePath}'.")
def asyncGenLogs(file):
EVTx = {}
json_data_dir = createDir('web\json_data')
print('EVTxFILE:', file)
fname = filename(file)
_df = pd.DataFrame()
parser = PyEvtxParser(file)
i = 0 # Register counter
for record in parser.records_json():
data = json.loads(record['data'])['Event']
readXmlObj(data, EVTx)
try:
df = pd.DataFrame.from_dict(EVTx).fillna(0)
_df = _df.append(df)
i += 1
except Exception as e:
pass
EVTx = {}
if i > (REG_COUNT_LIMIT - 1):
break
if not _df.empty:
if REGEX_PATERN:
try:
_df = _df[_df.apply(lambda x: x.astype(str).str.contains(
REGEX_PATERN, case=False, na=False)).any(axis=1)]
except Exception as e:
# error(e) #verbose
pass
_df = _df.set_index('EventRecordID').sort_values(by=['EventRecordID'],
ascending=False)
_df.to_json(f'{json_data_dir}\\{md5(fname)}.json', orient="table")
r, c = _df.shape
eel.feedback(f'Lines: {r}, Columns: {c}', md5(file))
else:
pd.DataFrame(['Empty'],
columns=['File'
]).to_json(f'{json_data_dir}\\{md5(fname)}.json',
index=False,
orient="table")
eel.feedback(f'Empty', md5(file))
def load_events(log_file_name):
"""
Opens Sysmon logs as a readable file, and turns the events into dictionaries
:param log_file_name: Sysmon evtx or xml log to be opened
:return: dict of the Sysmon event log
"""
try:
with open(log_file_name, "r", encoding='utf-8', errors='ignore') as fp:
magic = fp.read(7)
fp.seek(0)
if magic == 'ElfFile':
# log is evtx type
parser = PyEvtxParser(log_file_name)
dictrecords = [json.loads(rec['data']) for rec in parser.records_json()]
return dictrecords, 'evtx'
elif magic == '<Events':
return xmltodict.parse(fp.read()), 'xml'
else:
raise Exception('Invalid File magic')
except ExpatError:
raise KeyError("Error: Format error in the Event log file")
class Evtx2es(object):
def __init__(self, filepath: str) -> None:
self.path = Path(filepath)
self.parser = PyEvtxParser(self.path.open(mode='rb'))
def gen_json(self, size: int) -> Generator:
buffer: List[dict] = []
for record in self.parser.records_json():
record['data'] = json.loads(record.get('data'))
eventid_field = record.get('data').get('Event').get('System').get(
'EventID')
if type(eventid_field) is dict:
record['data']['Event']['System'][
'EventID'] = eventid_field.get('#text')
try:
status = record.get('data').get('Event').get('EventData').get(
'Status')
record['data']['Event']['EventData']['Status'] = None
except Exception:
pass
# Convert data according to ECS (sort of)
# First copy system fields
record['winlog'] = {
'channel':
record['data']['Event']['System']['Channel'],
'computer_name':
record['data']['Event']['System']['Computer'],
'event_id':
record['data']['Event']['System']['EventID'],
'opcode':
record['data']['Event']['System'].get('Opcode'),
'provider_guid':
record['data']['Event']['System']['Provider']
['#attributes'].get('Guid'),
'provider_name':
record['data']['Event']['System']['Provider']['#attributes']
['Name'],
'record_id':
record['data']['Event']['System']['EventRecordID'],
'task':
record['data']['Event']['System']['Task'],
'version':
record['data']['Event']['System'].get('Version'),
}
try:
record['winlog']['process'] = {
'pid':
record['data']['Event']['System']['Execution']
['#attributes']['ProcessID'],
'thread_id':
record['data']['Event']['System']['Execution']
['#attributes']['ThreadID'],
}
except KeyError:
pass
record.update({
'log': {
'file': {
'name': str(self.path)
}
},
'event': {
'code':
record['winlog']['event_id'],
'created':
record['data']['Event']['System']['TimeCreated']
['#attributes']['SystemTime'],
}
})
record['@timestamp'] = record['event']['created']
# Move event attributes to ECS location
record['winlog']['event_data'] = record['data']['Event'].get(
'EventData', dict())
del record['data']
if record['winlog']['event_data'] is None or len(
record['winlog']
['event_data']) == 0: # remove event_data fields if empty
del record['winlog']['event_data']
else:
for k, v in record['winlog']['event_data'].items():
# Normalize some known problematic fields with values switching between integers and strings with hexadecimal notation to integers
if k in ('ProcessId') and type(v) == str:
if v.startswith("0x"):
record['winlog']['event_data'][k] = int(v, 16)
else:
try:
record['winlog']['event_data'][k] = int(v)
except ValueError:
record['winlog']['event_data'][k] = 0
# Maximum limit of numeric values in Elasticsearch
if type(v) is int:
if v < -2**63:
record['winlog']['event_data'][k] = -2**63
elif v > 2**63 - 1:
record['winlog']['event_data'][k] = 2**63 - 1
buffer.append(record)
if len(buffer) >= size:
yield buffer
buffer.clear()
else:
yield buffer
def rdpevtx(infile, outdir):
outfilenametmp = os.path.basename(infile) + ".csv"
drive, outfiledir = os.path.splitdrive(os.path.dirname(infile))
#Make Dir
harddir = outdir + os.sep + outfiledir
outfilename = harddir + os.sep + outfilenametmp
print(f'Out Info: {outfilename}')
os.makedirs(harddir, exist_ok=True)
header = "'EventID'|'ComputerName'|'TimeCreated'|'Security SID'|'UserID'|'Address'|'SessionID/Logon Type/Reason/IP:Port'|'Event'|'Channel (EVTX Log)'|'SRC File'\n"
#create file
outfile = open(outfilename, 'w')
outfile.write(header)
parser = PyEvtxParser(infile)
print(f'Working on: {infile}')
for record in parser.records_json():
subrec = json.loads(record["data"])
lineout = ""
eventid = subrec['Event']['System']['EventID']
channel = subrec['Event']['System']['Channel']
computer = subrec['Event']['System']['Computer']
systemtime = subrec['Event']['System']['TimeCreated']['#attributes'][
'SystemTime']
#print(subrec['Event']['UserData']['EventXML'].keys())
#printjson(subrec)
if eventid == 21 or eventid == 22 or eventid == 23 or eventid == 24 or eventid == 25:
if channel == "Microsoft-Windows-TerminalServices-LocalSessionManager/Operational":
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'|" #1
lineout = lineout + "'" + computer + "'|" #2
lineout = lineout + "'" + systemtime + "'|" #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|" #4
lineout = lineout + "'" + subrec['Event']['UserData'][
'EventXML']['User'] + "'|" #5
#Shitty way to get the value of Address from JSON but could not get it pulled any other way
#for x in retrieve_nested_value(subrec, "Address"):
# lineout = lineout + "'From: " + x + "'|"
if 'Address' in subrec['Event']['UserData']['EventXML'].keys():
address = (
subrec['Event']['UserData']['EventXML']['Address'])
else:
address = '' #or empty string
lineout = lineout + "'From: " + address + "'|" #6
lineout = lineout + "'" + str(
subrec['Event']['UserData']['EventXML']
['SessionID']) + "'|" #7
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8
lineout = lineout + "'" + channel + "'|" #9
lineout = lineout + "'" + infile + "'" #10
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 39:
if channel == "Microsoft-Windows-TerminalServices-LocalSessionManager/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|" #1
lineout = lineout + "'" + computer + "'|" #2
lineout = lineout + "'" + systemtime + "'|" #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|" #4
lineout = lineout + "|" #5
lineout = lineout + "|" #6
#lineout = lineout + "'" + "Session: " + str(subrec['Event']['UserData']['EventXML']['Session'])
#lineout = lineout + " Reason: " + rdpdisconnect[str(subrec['Event']['UserData']['EventXML']['Reason'])] + "'|"
lineout = lineout + "'" + "Session: " + str(
subrec['Event']['UserData']['EventXML']
['TargetSession']) #7
lineout = lineout + " Source: " + str(
subrec['Event']['UserData']['EventXML']
['Source']) + "'|" #7
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8
lineout = lineout + "'" + channel + "'|" #9
lineout = lineout + "'" + infile + "'" #10
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 40:
if channel == "Microsoft-Windows-TerminalServices-LocalSessionManager/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|" #1
lineout = lineout + "'" + computer + "'|" #2
lineout = lineout + "'" + systemtime + "'|" #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|" #4
lineout = lineout + "|" #5
lineout = lineout + "|" #6
lineout = lineout + "'" + "Session: " + str(
subrec['Event']['UserData']['EventXML']['Session']) #7
lineout = lineout + " Reason: " + rdpdisconnect[str(
subrec['Event']['UserData']['EventXML']
['Reason'])] + "'|" #7
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8
lineout = lineout + "'" + channel + "'|" #9
lineout = lineout + "'" + infile + "'" #10
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
#--------------------------------------------------------------------------------------------------------------------------------
if eventid == 261:
if channel == "Microsoft-Windows-TerminalServices-RemoteConnectionManager/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 1149:
if channel == "Microsoft-Windows-TerminalServices-RemoteConnectionManager/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "'" + subrec['Event']['UserData'][
'EventXML']['Param2'] + "\\" + subrec['Event']['UserData'][
'EventXML']['Param1'] + "'|"
lineout = lineout + "'" + "Src IP: " + subrec['Event'][
'UserData']['EventXML']['Param3'] + "'|"
lineout = lineout + "|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
#---------------------------------------------------------------------------------------------------------------------------------
if eventid == 131 or eventid == 140:
if channel == "Microsoft-Windows-RemoteDesktopServices-RdpCoreTS/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "'From: " + subrec['Event']['EventData'][
'ClientIP'] + "'|"
lineout = lineout + "|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
#---------------------------------------------------------------------------------------------------------------------------------
if eventid == 4624:
if channel == "Security":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|" #1
lineout = lineout + "'" + computer + "'|" #2
lineout = lineout + "'" + systemtime + "'|" #3
lineout = lineout + "|" #Security UserID #4
# 6 TargetDomainname
# 5 TargetUserName
# 4 TargetUserSid
# 8 Logon Type
# 11 Workstation Name
# 18 IPaddress
# 19 Port
lineout = lineout + "'" + subrec['Event']['EventData'][
'TargetDomainName'] + "\\" + subrec['Event']['EventData'][
'TargetUserName']
lineout = lineout + " (" + subrec['Event']['EventData'][
'TargetUserSid'] + ") " + "'|" #5
lineout = lineout + "'" + subrec['Event']['EventData'][
'IpAddress'] + ":" + subrec['Event']['EventData'][
'IpPort'] + " -> "
lineout = lineout + subrec['Event']['EventData'][
'WorkstationName'] + "'|" #6
lineout = lineout + "'Logon Type: " + str(
subrec['Event']['EventData']['LogonType']) + "'|" #7
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8
lineout = lineout + "'" + channel + "'|" #9
lineout = lineout + "'" + infile + "'" #10
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 4625:
if channel == "Security":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "|" #Security UserID
# 6 TargetDomainname
# 5 TargetUserName
# 4 TargetUserSid
# 8 FailureReason
# 10 LogonType
# 13 WorkstationName
# 19 IPaddress
# 20 Port
lineout = lineout + "'" + subrec['Event']['EventData'][
'TargetDomainName'] + "\\" + subrec['Event']['EventData'][
'TargetUserName']
lineout = lineout + " (" + subrec['Event']['EventData'][
'TargetUserSid'] + ") " + "'|"
lineout = lineout + "'From: " + subrec['Event']['EventData'][
'WorkstationName'] + " (" + subrec['Event']['EventData'][
'IpAddress'] + ":" + subrec['Event']['EventData'][
'IpPort'] + ")'|"
lineout = lineout + "'Logon Type: " + str(
subrec['Event']['EventData']
['LogonType']) + " Failure Reason: " + subrec['Event'][
'EventData']['FailureReason'] + "'|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 4634:
if channel == "Security":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|" #1
lineout = lineout + "'" + computer + "'|" #2
lineout = lineout + "'" + systemtime + "'|" #3
lineout = lineout + "|" #Security UserID #4
# 3 TargetDomainname
# 1 TargetUserName
# 0 TargetUserSid
# 4 LogonType
lineout = lineout + "'" + subrec['Event']['EventData'][
'TargetDomainName'] + "\\" + subrec['Event']['EventData'][
'TargetUserName']
lineout = lineout + " (" + subrec['Event']['EventData'][
'TargetUserSid'] + ") " + "'|" #5
lineout = lineout + "|" #6
lineout = lineout + "'Logon Type: " + str(
subrec['Event']['EventData']['LogonType']) + "'|" #7
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8
lineout = lineout + "'" + channel + "'|" #9
lineout = lineout + "'" + infile + "'" #10
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 4647:
if channel == "Security":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|" #EventID
lineout = lineout + "'" + computer + "'|" #ComputerName
lineout = lineout + "'" + systemtime + "'|" #Time Created
lineout = lineout + "|" #Security UserID #4
# 3 TargetDomainname
# 1 TargetUserName
# 0 TargetUserSid
# 4 LogonType
lineout = lineout + "'" + subrec['Event']['EventData'][
'TargetDomainName'] + "\\" + subrec['Event']['EventData'][
'TargetUserName']
lineout = lineout + " (" + subrec['Event']['EventData'][
'TargetUserSid'] + ") " + "'|" #UserID
lineout = lineout + "|" #Address
lineout = lineout + "|" #SessionID/Logon Type/Reason/IP:Port
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|" #8
lineout = lineout + "'" + channel + "'|" #9
lineout = lineout + "'" + infile + "'" #10
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 4648:
if channel == "Security":
if search('winlogon.exe$',
subrec['Event']['EventData']['ProcessName']):
#printjson(subrec)
lineout = "'" + str(eventid) + "'|" #EventID
lineout = lineout + "'" + computer + "'|" #ComputerName
lineout = lineout + "'" + systemtime + "'|" #Time Created
lineout = lineout + "|" #Security UserID
# 6 TargetDomainname
# 5 TargetUserName
# 8 TargetServerName
# 11 Processname
# 12 IPaddress
# 13 Port
lineout = lineout + "'Subject: " + subrec['Event'][
'EventData']['SubjectDomainName'] + "\\" + subrec[
'Event']['EventData']['SubjectUserName']
lineout = lineout + " (LogonID: " + subrec['Event'][
'EventData']['SubjectUserSid'] + ") "
lineout = lineout + "Target: " + subrec['Event'][
'EventData']['TargetDomainName'] + "\\" + subrec[
'Event']['EventData'][
'TargetUserName'] + "'|" #UserID
lineout = lineout + "|'" + subrec['Event']['EventData'][
'IpAddress'] + ":" + subrec['Event']['EventData'][
'IpPort']
lineout = lineout + " Target: " + subrec['Event'][
'EventData']['TargetServerName'] + " (" + subrec[
'Event']['EventData'][
'TargetInfo'] + ") " + "'|" #Address
lineout = lineout + "|'Process: " + subrec['Event'][
'EventData']['ProcessName'] + " (" + subrec['Event'][
'EventData'][
'ProcessId'] + ")'|" #SessionID/Logon Type/Reason/IP:Port
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 4778 or eventid == 4779:
if channel == "Security":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|" #EventID
lineout = lineout + "'" + computer + "'|" #ComputerName
lineout = lineout + "'" + systemtime + "'|" #Time Created
lineout = lineout + "|" #Security UserID
# 3 TargetDomainname
# 1 TargetUserName
# 0 TargetUserSid
lineout = lineout + "'" + subrec['Event']['EventData'][
'AccountDomain'] + "\\" + subrec['Event']['EventData'][
'AccountName']
lineout = lineout + " (LogonID: " + subrec['Event'][
'EventData']['LogonID'] + ") " + "'|" #UserID
lineout = lineout + "|'" + subrec['Event']['EventData'][
'ClientName'] + " (" + subrec['Event']['EventData'][
'ClientAddress'] + ") " + "'|" #Address
lineout = lineout + "|'" + subrec['Event']['EventData'][
'SessionName'] + "'|" #SessionID/Logon Type/Reason/IP:Port
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
#---------------------------------------------------------------------------------------------------------------------------------
if eventid == 1024:
if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "'Connect to: " + subrec['Event'][
'EventData']['Value'] + "'|"
lineout = lineout + "|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 1025:
if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 1026:
if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + subrec['Event']['EventData'][
'CustomLevel'] + ": " + subrec['Event']['EventData'][
'Name'] + " " + str(
subrec['Event']['EventData']['Value']) + "'|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
if eventid == 1029:
if channel == "Microsoft-Windows-TerminalServices-RDPClient/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "Userid Base64 Hash: " + subrec['Event'][
'EventData']['TraceMessage'] + "'|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
#---------------------------------------------------------------------------------------------------------------------------------
if eventid == 56:
if channel == "System":
printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
#System 9009 Eventid
#if '#text' in subrec['Event']['System']['EventID'].keys():
# eventidsys = subrec['Event']['System']['EventID']['#text']
#else:
# eventidsys = 0 #or empty string
#if channel == "System":
#print(f'{subrec['Event']['System'].keys()}')
# print(subrec['Event']['UserData']['EventXML'].keys())
#if eventidsys == "9009":
# if channel == "System":
# if subrec['Event']['System']['Provider']['attributes']['Name'] == 'Desktop Window Manager':
# printjson(subrec)
# lineout = "'" + str(eventid) + "'|"
# lineout = lineout + "'" + computer + "'|"
# lineout = lineout + "'" + systemtime + "'|"
# lineout = lineout + "'" + subrec['Event']['System']['Security']['#attributes']['UserID'] + "'|"
# lineout = lineout + "|"
# lineout = lineout + "|"
# lineout = lineout + "|"
# lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
# lineout = lineout + "'" + channel + "'|"
# lineout = lineout + "'" + infile + "'"
# lineout = lineout + "|python"
# lineout = lineout + "\n"
#print(f'{lineout}')
# outfile.write(lineout)
#---------------------------------------------------------------------------------------------------------------------------------
if eventid == 7001 or eventid == 7002:
if channel == "Microsoft-Windows-Winlogon/Operational":
#printjson(subrec)
lineout = "'" + str(eventid) + "'|"
lineout = lineout + "'" + computer + "'|"
lineout = lineout + "'" + systemtime + "'|"
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes']['UserID'] + "'|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "|"
lineout = lineout + "'" + eventidtxt[str(eventid)] + "'|"
lineout = lineout + "'" + channel + "'|"
lineout = lineout + "'" + infile + "'"
lineout = lineout + "|python"
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
#print(subrec)
#print(subrec['Event']['UserData']['EventXML'])
#print(subrec['Event']['UserData']['EventXML']['Address'])
#print(f'------------------------------------------')
print("end")
outfile.close()
<head>
<meta charset="utf-8" />
<title> Report </title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" type="text/css" media="screen" href="style.css" />
</head>
<body>
<div class="wrapper">
<div class="header">
<H1> Report </H1>
</div>'''
html_code += f'\n<table align="center"> \n <caption><h2><b>Partition%4Diagnostic.evtx ANALYSIS REPORT for device with S/N: {serial}</b></h2></caption>'
html_code += '\n<tr style="background-color:DarkGrey"> \n <th>EventRecordID</th> \n <th>connected Timestamp (UTC)</th> \n <th>Manufacturer</th> \n <th>Model</th> \n <th>Volume 1 Serial Number</th> \n <th>Volume 2 Serial Number</th> \n <th>Volume 3 Serial Number</th> \n <th>Volume 4 Serial Number</th> \n <th>Flag</th>'
for record in parser.records_json():
data = json.loads(record['data'])
records_dict[(data['Event']['System']['EventRecordID'])] = data #add gia na sortaro meta me event id kai na exo xronologika sosti seira
if data['Event']['System']['EventID'] != 1006:
IsPartitionDiagnosticEVTX = False
if IsPartitionDiagnosticEVTX: #parsarw giati einai to PartitionDiagnostic evtx afou ola ta EventID einai 1006
print('Initializing parsing')
print('.................')
LogStartTime = records_dict[1]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC')
LogEndTime = records_dict[len(records_dict)]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC')
print('Parsing complete')
print('.................')
print('Initializing analysis')
print('.................')
for i in sorted(records_dict.keys()):
def ProcessPrivateRules(self, logspath):
if self.logger:
self.logger.info(
f"Starting processing private rules on the log/s in '{logspath}' ..."
)
for pubrule in self.PublicRulesContainsPrivateRules:
triggered = None
privRules = []
privRulesChannels = []
TriggeredEvents = {}
for privrulename in pubrule.include.get("rule"):
for rule in self.ruleSet:
if rule.name == privrulename:
privRules.append(rule)
privRulesChannels.append(rule.channel.lower())
for filePath in self.LogsToProcess:
parser = PyEvtxParser(filePath)
for record in parser.records_json():
try:
data = json.loads(record["data"])
event = Event(data)
if event.Channel.lower() in privRulesChannels:
for prirule in privRules:
if self.match(prirule, event):
if triggered == None:
triggered = True
triggered = triggered and True
if not TriggeredEvents.get(prirule):
TriggeredEvents[prirule] = [event]
else:
TriggeredEvents[prirule].append(event)
else:
break
except (OSError, KeyError) as e:
if self.logger:
self.logger.error(e, exc_info=True)
continue
except Exception as e:
if self.logger:
self.logger.error(e, exc_info=True)
if len(TriggeredEvents) != len(privRules):
return False
TriggeredEventsList = []
for key, val in TriggeredEvents.items():
TriggeredEventsList.append(val)
TriggeredEventsWithinTheSpecifiedTime = self.ProcessTimeBetweenLogs(
list(itertools.product(*TriggeredEventsList)),
int(pubrule.include.get("if").get("within")))
if TriggeredEventsWithinTheSpecifiedTime:
for EventSet in TriggeredEventsWithinTheSpecifiedTime:
recordIDs = []
triggeredEventsData = {}
privRuleNames = pubrule.include.get("rule")
privateRules = []
for r in self.ruleSet:
if r.name in privRuleNames:
privateRules.append(r)
for r in privateRules:
for e in EventSet:
if r.channel == e.Channel:
if r.returns:
fields = {}
for field in r.returns:
fields.update(
{field: e.EventData.get(field)})
triggeredEventsData[r.name] = fields
else:
triggeredEventsData[r.name] = e.RawRecord
for event in EventSet:
recordIDs.append(event.EventRecordID)
data = [
event.TimeCreatedSystemTime, recordIDs, pubrule.name,
pubrule.score, pubrule.description, pubrule.reference,
[], triggeredEventsData
]
self.Queue.put(
Alert(event,
pubrule, [],
privateRule=True,
record=data))
else:
return False
def fullparse():
filename = values['-IN-']
FullParseRecordsDict = {}
AllPluggedInSerials = []
EachPluggedDeviceDict = {}
IsPartitionDiagnosticEVTXFullParse = True
FullParseHTMLWritten = True
try: #checkarw an einai legit evtx log
parser = PyEvtxParser(filename)
for record in parser.records_json():
data = json.loads(record['data'])
FullParseRecordsDict[(data['Event']['System']['EventRecordID'])] = data #add gia na sortaro meta me event id kai na exo xronologika sosti seira
if data['Event']['System']['EventID'] != 1006:
IsPartitionDiagnosticEVTXFullParse = False
FullParseLogStartTime = FullParseRecordsDict[1]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC')
FullParseLogEndTime = FullParseRecordsDict[len(FullParseRecordsDict)]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC')
if IsPartitionDiagnosticEVTXFullParse:
for i in sorted(FullParseRecordsDict.keys()):
if FullParseRecordsDict[i]['Event']['EventData']['SerialNumber'] not in AllPluggedInSerials: # prwth fora pou vrskv to usb sto log opote to grafw sigoura
AllPluggedInSerials.append(FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']) # pros8etw sth lista to S/N tou usb
if FullParseRecordsDict[i]['Event']['EventData']['PartitionStyle'] == 1: #an einai GPT schemed media
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']] = [FullParseRecordsDict[i]['Event']['EventData']['Manufacturer'], FullParseRecordsDict[i]['Event']['EventData']['Model'], FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), ['GPT - NO VSN INFO']] # EachPluggedDeviceDict = {'serial':[manufacturer, model, first pluggedIn, lastpluggedIn, [VSN1, VSN2 klp]]
else:
SN0 = volumeSNParser(FullParseRecordsDict[i], 'Vbr0', FullParseRecordsDict[i]['Event']['EventData']['Vbr0'][6:18])
if FullParseRecordsDict[i]['Event']['EventData']['Vbr1'] != '':
SN1 = volumeSNParser(FullParseRecordsDict[i], 'Vbr1', FullParseRecordsDict[i]['Event']['EventData']['Vbr1'][6:18])
else:
SN1 = '-'
if FullParseRecordsDict[i]['Event']['EventData']['Vbr2'] != '':
SN2 = volumeSNParser(FullParseRecordsDict[i], 'Vbr2', FullParseRecordsDict[i]['Event']['EventData']['Vbr2'][6:18])
else:
SN2 = '-'
if FullParseRecordsDict[i]['Event']['EventData']['Vbr3'] != '':
SN3 = volumeSNParser(FullParseRecordsDict[i], 'Vbr3', FullParseRecordsDict[i]['Event']['EventData']['Vbr3'][6:18])
else:
SN3 = '-'
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']] = [FullParseRecordsDict[i]['Event']['EventData']['Manufacturer'], FullParseRecordsDict[i]['Event']['EventData']['Model'], FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC'), [SN0, SN1, SN2, SN3]]
else: #to exw ksanavrei to S/N tou usb opote einai hdh sto dict kai apla 8elw na kanvw update to timestamp kai ta VSN
if FullParseRecordsDict[i]['Event']['EventData']['PartitionStyle'] == 1: #an einai GPT schemed media opote sigoua den exei allaksei VSN afou den ta deixnei
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][3] = FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC') # kanw update mono to last pluggedin time
else: #einai eite GPt pou egine unplugged h' MBR
if FullParseRecordsDict[i]['Event']['EventData']['Vbr0'] != '': #VBR0 oxi keno opote exw vrei MBR plugged in record
SN0 = volumeSNParser(FullParseRecordsDict[i], 'Vbr0', FullParseRecordsDict[i]['Event']['EventData']['Vbr0'][6:18])
if FullParseRecordsDict[i]['Event']['EventData']['Vbr1'] != '':
SN1 = volumeSNParser(FullParseRecordsDict[i], 'Vbr1', FullParseRecordsDict[i]['Event']['EventData']['Vbr1'][6:18])
else:
SN1 = '-'
if FullParseRecordsDict[i]['Event']['EventData']['Vbr2'] != '':
SN2 = volumeSNParser(FullParseRecordsDict[i], 'Vbr2', FullParseRecordsDict[i]['Event']['EventData']['Vbr2'][6:18])
else:
SN2 = '-'
if FullParseRecordsDict[i]['Event']['EventData']['Vbr3'] != '':
SN3 = volumeSNParser(FullParseRecordsDict[i], 'Vbr3', FullParseRecordsDict[i]['Event']['EventData']['Vbr3'][6:18])
else:
SN3 = '-'
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][3] = FullParseRecordsDict[i]['Event']['System']['TimeCreated']['#attributes']['SystemTime'].replace('T', ' ').replace('Z', ' UTC')
if SN3 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN3)
if SN2 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN2)
if SN1 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN1)
if SN0 not in EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4]: #checkarw an einai hdh sth lista me ta VSN tou media alliws to pros8etw
EachPluggedDeviceDict[FullParseRecordsDict[i]['Event']['EventData']['SerialNumber']][4].append(SN0)
else:
continue #next iteration giati exw vrei record eite gia MBR eite GPT alla pou einai gia unplugged
FullParsehtml_code ='''<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title> Report </title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" type="text/css" media="screen" href="style.css" />
</head>
<body>
<div class="wrapper">
<div class="header">
<H1> Report </H1>
</div>'''
FullParsehtml_code += f'\n<table align="center"> \n <caption><h2><b>Full Report for all connected devices</b></h2></caption>'
FullParsehtml_code += '\n<tr style="background-color:DarkGrey"> \n <th>Media S/N</th> \n <th>Manufacturer</th> \n <th>Model</th> \n <th>First connected Timestamp (UTC)</th> \n <th>Last connected Timestamp (UTC)</th> \n <th>Every VSN recovered from the log</th>'
for serial in AllPluggedInSerials:
FullParsehtml_code += f'\n<tr> \n <td>{serial}</td> \n<td>{EachPluggedDeviceDict[serial][0]}</td> \n <td>{EachPluggedDeviceDict[serial][1]}</td> \n<td>{EachPluggedDeviceDict[serial][2]}</td> \n<td>{EachPluggedDeviceDict[serial][3]} </td>\n <td>'
for i in range(len(EachPluggedDeviceDict[serial][4])):
if EachPluggedDeviceDict[serial][4][i] == '-' or EachPluggedDeviceDict[serial][4][i] == 'Unknown Volume Type':
continue
FullParsehtml_code += f'{EachPluggedDeviceDict[serial][4][i]} '
FullParsehtml_code += '</td>'
FullParsehtml_code += f'\n</table> \n<br>\n<p align="left" style="color:white">\n <u style="font-size:20px"> Complete Log Timeline</u> <br>\n From: {FullParseLogStartTime} <br>\nTo: {FullParseLogEndTime} <br>\n</p> \n<br> <br> <br>\n<div class="push"></div> \n</div> \n <div class="footer">--Partition%4DiagnosticParser Ver. 1.3.0</div>\n</body>\n</html>'
FullParsecss_code = '''table{border-collapse:collapse;}
th{text-align:center;background-color:#4a4343;color=white;}
table,th,td{border:1px solid #000;}
tr{text-align:center;background-color:#595555; color:white;}
html, body {
height: 100%;
margin: 0;
}
.wrapper {
min-height: 100%;
background-color: #4a4349;
/* Equal to height of footer */
/* But also accounting for potential margin-bottom of last child */
margin-bottom: -50px;
font-family: "Courier New", sans-serif;
color=white;
}
.header{
background-color: dark grey;
color=white;
}
.header h1 {
text-align: center;
font-family: "Courier New", sans-serif;
color=red;
}
.push {
height: 50px;
background-color: #4a4349;
}
.footer {
height: 50px;
background-color: #4a4349;
color=white;
text-align: right;
} '''
if values['-INSAVE-'] == defaultOutputPathText: #dhl tickare gia html report alla den epelekse fakelo gia save alla afise to default
try:
with open('Full_Report.html', 'w', encoding='utf8') as fout:
fout.write(FullParsehtml_code)
with open('style.css', 'w', encoding= 'utf8') as cssout:
cssout.write(FullParsecss_code)
except:
FullParseHTMLWritten = False
else: #dhl exei dwsei output save folder
try:
with open(f"{values['-INSAVE-']}/Full_Report.html", 'w', encoding='utf8') as fout:
fout.write(FullParsehtml_code)
with open(f"{values['-INSAVE-']}/style.css", 'w', encoding= 'utf8') as cssout:
cssout.write(FullParsecss_code)
except:
FullParseHTMLWritten = False
if FullParseHTMLWritten:
if values['-DISKSN-'] == '':
print('Initializing parsing')
print('.................')
print('Parsing complete')
print('.................')
print('Initializing analysis')
print('.................')
print('Analysis complete')
print('.................')
print('---------------------------')
print('Analysis Results')
print('---------------------------')
print()
print()
print(f'Log timeline:')
print(f'Start: {FullParseLogStartTime}')
print(f'End: {FullParseLogEndTime}')
print()
print('Full analysis report for all connected devices completed')
sg.PopupOK('Full report created succesfully!', title=':)', background_color='#2a363b')
else:
print('Full report has been created succesfully but was unable to be written to disk\nCheck write permissions for the selected output folder!')
else:
sg.PopupOK('Not a PartitionDiagnostic EVTX log file!', title='!!!', background_color='#2a363b')
window['-IN-'].update('')
window['-DISKSN-'].update('')
except Exception as e:
print(e)
sg.PopupOK('Error parsing the chosen EVTX log file!', title='Error', background_color='#2a363b')
window['-IN-'].update('')
window['-DISKSN-'].update('')
def driver(case_name, file_name, Artifact_ID):
conn = pymysql.connect(host='192.168.4.188',
user='******',
password='******',
db=case_name,
charset='utf8')
curs = conn.cursor()
sql_usb_log = "insert into Usb_Event_Log (Artifact_ID, Event_ID,System_Time, Serial_Number, Manufacturer,Usb_Name) values (%s, %s, %s, %s, %s, %s)"
file = file_name
parser = PyEvtxParser(file)
id = 0
try:
for record in parser.records_json():
id += 1
event = json.loads(record['data'])['Event']
try:
Event_ID = event['System']['EventID']
except:
Event_ID = None
print("Event_ID Error")
try:
timestamp = record['timestamp']
except:
timestamp = None
print("Timestamp Error")
try:
Instance_ID = event['UserData']['UMDFHostDeviceRequest'][
'InstanceId']
if Instance_ID.startswith('USB'):
try:
serial = Instance_ID.split('\\')[2]
except:
serial = Instance_ID
try:
prod = Instance_ID.split('\\')[1].split('&')[0].split(
'VID_')[1]
except:
prod = None
try:
ven = Instance_ID.split('\\')[1].split('&')[1].split(
'PID_')[1]
except:
ven = None
try:
t1 = datetime.strptime(timestamp,
'%Y-%m-%d %H:%M:%S.%f UTC')
t1.strftime('%Y-%m-%d %H:%M:%S')
except:
t1 = datetime.datetime(1900, 1, 1, 00, 00, 00)
elif Instance_ID.startswith('SWD'):
try:
serial = Instance_ID.split('#')[2].split('&')[0]
except:
serial = reg.search(Instance_ID)[0]
try:
ven = Instance_ID.split('&')[1].split('VEN_')[1]
except:
ven = None
try:
prod = Instance_ID.split('&')[2].split('PROD_')[1]
except:
prod = None
try:
t1 = datetime.strptime(timestamp,
'%Y-%m-%d %H:%M:%S.%f UTC')
t1.strftime('%Y-%m-%d %H:%M:%S')
except:
t1 = datetime.datetime(1900, 1, 1, 00, 00, 00)
else:
continue
except:
print("Instance_ID Error")
curs.execute(sql_usb_log,
(Artifact_ID, Event_ID, t1, serial, ven, prod))
except:
pass
conn.commit()
conn.close()
def partition(case_name, file_name, Artifact_ID):
conn = pymysql.connect(host='192.168.4.188',
user='******',
password='******',
db=case_name,
charset='utf8')
curs = conn.cursor()
file = file_name
sql_partition_log = "insert into Usb_Partition_Log(Artifact_ID,Manufacturer , Model, Serial_Number , VSN) values (%s, %s, %s, %s, %s)"
parser = PyEvtxParser(file)
instance = 0
id = 0
table = []
try:
for record in parser.records_json():
id += 1
data = json.loads(record['data'])['Event']
try:
Manufacture = data['EventData']['Manufacturer']
except:
Manufacture = None
print("Manufacture error")
try:
Model = data['EventData']['Model']
except:
Model = None
print("Model error")
try:
ParentId = data['EventData']['ParentId']
Serial_Number = ParentId.split('\\')[2]
except:
Serial_Number = None
print("Serial Number error")
try:
VBR = data['EventData']['Vbr0']
if VBR[6:14] == '4e544653':
vsn_reverse = VBR[144:152] #NTFS
vsn = '0x'
count = 9
for index in range(4):
count -= 2
vsn = vsn + vsn_reverse[count - 1:count + 1]
elif VBR[164:174] == '4641543332':
vsn_reverse = VBR[134:142] # FAT32
vsn = '0x'
count = 9
for index in range(4):
count -= 2
vsn = vsn + vsn_reverse[count - 1:count + 1]
elif VBR[6:16] == '4558464154':
vsn_reverse = VBR[200:208] # exfat
vsn = '0x'
count = 9
for index in range(4):
count -= 2
vsn = vsn + vsn_reverse[count - 1:count + 1]
elif VBR[108:116] == '46415431':
vsn_reverse = VBR[78:86] # fat16
vsn = '0x'
count = 9
for index in range(4):
count -= 2
vsn = vsn + vsn_reverse[count - 1:count + 1]
else:
vsn = None
except:
vsn = None
print("vsn error")
if Manufacture != None and Model != None and Serial_Number != None and vsn != None:
table.append([Manufacture, Model, Serial_Number, vsn])
new_table = []
for element in table:
if element not in new_table:
new_table.append(element)
for column in new_table:
curs.execute(
sql_partition_log,
(Artifact_ID, column[0], column[1], column[2], column[3]))
except:
print("sql error")
conn.commit()
conn.close()
def nom_file(filename, welm_map):
parser = PyEvtxParser(filename)
# Open Records
for record in parser.records_json():
data = json.loads(record['data'])
try:
# Event Log event
event = {'recordid': str(record['event_record_id'])}
event.update(get_section(data['Event']['System']))
if data['Event'].get('EventData'):
event['event_data'] = get_section(data['Event']['EventData'])
if data['Event'].get('UserData'):
#print(data['Event'].get('UserData'))
if data['Event']['UserData'].get('EventXML'):
event['event_data'] = get_section(
data['Event']['UserData']['EventXML'])
else:
# not sure about what other namesspaces are here so for now just this loop
for ns in data['Event']['UserData']:
event['event_data'] = get_section(
data['Event']['UserData'][ns])
if isinstance(event['eventid'], dict):
print(event['eventid'])
print("#" * 20)
print(json.dumps(event, indent=3))
print("#" * 20)
print(json.dumps(data, indent=3))
key = make_key(
event.get('channel') or '', event['provider']['name'],
event['eventid'])
if key in welm_map:
if welm_map[key][
'swap_mode'] and welm_map[key]['params'] != []:
if event.get('event_data') or False:
swap_target = 'event_data'
elif event.get('user_data') or False:
swap_target = 'user_data'
else:
swap_target = None
event['message'] = welm_map[key]['format_string']
if swap_target:
swap_values = ['bump']
for param in welm_map[key]['params']:
swap_values.append(event[swap_target].get(param)
or "")
#print(key)
#print(welm_map[key]['format_string'])
#print(welm_map[key]['params'])
#print(swap_values)
try:
event['message'] = welm_map[key][
'format_string'].format(*swap_values)
except:
event['message'] = welm_map[key]['format_string']
else:
event['message'] = welm_map[key]['format_string']
else:
event[
'message'] = "{} | {} | {} | Unknown Message String".format(
event['eventid'],
event.get('channel') or '', event['provider']['name'])
# Raw Document
event['raw'] = record['data']
yield event
except KeyError as errormsg:
print("Soemthing went wrong parsing this event")
print(json.dumps(data, indent=4))
def parser_simple(path, opath=None):
import time
cursor = sql_connection()
sql_initialitation(cursor)
parser = PyEvtxParser(path)
connections_key = 100000000
thread_key = 0
files_inserted = []
full_process_inserted = []
pipes_inserted = []
threads_inserted = []
start = time.time()
for record in parser.records_json():
event = json.loads(record['data'])
# Process Creation
if event["Event"]["System"]["EventID"] == 1:
try:
if event["Event"]["EventData"]["ProcessGuid"] not in full_process_inserted:
#print("Full inserted")
query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image","IntegrityLevel",' \
'"TerminalSessionId", "User")' \
" VALUES ('{}','{}','{}','{}','{}', '{}')" \
' ON CONFLICT ("ProcessGuid") DO UPDATE SET "IntegrityLevel" = ' \
'EXCLUDED."IntegrityLevel", "TerminalSessionId"' \
' = EXCLUDED."TerminalSessionId", "User" = EXCLUDED."User";'.format(
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["ProcessId"],
event["Event"]["EventData"]["Image"],
event["Event"]["EventData"]["IntegrityLevel"],
event["Event"]["EventData"]["TerminalSessionId"],
event["Event"]["EventData"]["User"])
cursor.execute(query_process)
full_process_inserted.append(event["Event"]["EventData"]["ProcessGuid"])
except Exception as e:
logger.error("Error query_process 1: " + str(e) + " Event: " + str(event["Event"]))
try:
if event["Event"]["EventData"]["Image"] not in files_inserted:
if "OriginalFileName" in event["Event"]["EventData"]:
original = ",'" + str(event["Event"]["EventData"]["OriginalFileName"]) + "'"
key = ',"OriginalFileName"'
else:
original = ""
key = ""
except Exception as e:
logger.error("Error query_file 1: " + str(e) + " Event: " + str(event["Event"]))
try:
query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["ParentImage"]).lower(),
event["Event"]["EventData"]["ParentProcessId"],
event["Event"]["EventData"]["ParentImage"])
cursor.execute(query_pprocess)
except Exception as e:
logger.error("Error query_pprocess_exist 1: " + str(e) + " Event: " + str(query_pprocess))
try:
if event["Event"]["EventData"]["CommandLine"]:
event["Event"]["EventData"]["CommandLine"] = str(
event["Event"]["EventData"]["CommandLine"]).replace("'", "\"")
if event["Event"]["EventData"]["CurrentDirectory"]:
event["Event"]["EventData"]["CurrentDirectory"] = str(
event["Event"]["EventData"]["CurrentDirectory"]).replace("'", "\"")
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \
'"ExtraInfo","ExtraInfo2")' \
" VALUES ('{}','{}','{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"], "CreateProcess",
(event["Event"]["EventData"]["ParentImage"]).lower(),
event["Event"]["EventData"]["LogonGuid"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["CommandLine"],
event["Event"]["EventData"]["CurrentDirectory"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 1: " + str(e) + " Event: " + str(query_action))
try:
query_user = '******' \
" ('{}','{}','{}',{}) ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["LogonGuid"],
event["Event"]["EventData"]["User"],
event["Event"]["EventData"]["LogonId"],
event["Event"]["EventData"]["TerminalSessionId"])
cursor.execute(query_user)
except Exception as e:
logger.error("Error 1: " + str(e) + " Event: " + str(query_user))
# File creation time changed
if event["Event"]["System"]["EventID"] == 2:
logger.info("ToDo")
# Network connection
if event["Event"]["System"]["EventID"] == 3:
try:
connections_key = str(event["Event"]["EventData"]["SourceIp"])+str(event["Event"]["EventData"]["DestinationIp"])
query_connection = 'INSERT INTO public."Connections" ("ConnectionId","Protocol","SourceIp","SourceHostname",' \
'"SourcePort","DestinationIsIpv6","DestinationIp","DestinationHostname","DestinationPort") ' \
"VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format(
connections_key, event["Event"]["EventData"]["Protocol"],
event["Event"]["EventData"]["SourceIp"],
event["Event"]["EventData"]["SourceHostname"],
event["Event"]["EventData"]["SourcePort"],
event["Event"]["EventData"]["DestinationIsIpv6"],
event["Event"]["EventData"]["DestinationIp"],
event["Event"]["EventData"]["DestinationHostname"],
event["Event"]["EventData"]["DestinationPort"])
cursor.execute(query_connection)
except Exception as e:
logger.error("Error query_connection 3: " + str(e) + " Event: " + str(query_connection))
try:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \
'"ExtraInfo")' \
" VALUES ('{}','{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"], "CreateConnection",
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["User"],
connections_key,
event["Event"]["EventData"]["Initiated"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 3: " + str(e) + " Event: " + str(query_action))
try:
insert_process(cursor, event)
except Exception as e:
logger.error("Error query_pprocess_exist 3: " + str(e) + " Event: " + str(event["Event"]))
# Process Terminated
if event["Event"]["System"]["EventID"] == 5:
try: # Destination Process
query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["ProcessId"],
event["Event"]["EventData"]["Image"])
cursor.execute(query_process)
except Exception as e:
logger.error("Error query_sprocess 5: " + str(e) + " Event: " + str(query_process))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"ProcessTerminated",
(event["Event"]["EventData"]["Image"]).lower(),
(event["Event"]["EventData"]["Image"]).lower())
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 5: " + str(e) + " Event: " + str(query_action))
# Kernel driver loaded
if event["Event"]["System"]["EventID"] == 6:
logger.info("ToDo")
# Image loaded
if event["Event"]["System"]["EventID"] == 7:
try: # Process
insert_process(cursor, event)
except Exception as e:
logger.error("Error query_sprocess 7: " + str(e) + " Event: " + str(query_sprocess))
try: # File
if "Description" not in event["Event"]["EventData"]:
event["Event"]["EventData"]["Description"] = ""
if "OriginalFileName" not in event["Event"]["EventData"]:
event["Event"]["EventData"]["OriginalFileName"] = ""
query_file = 'INSERT INTO public."Files" ("Filename","FileVersion","Description","Product","Company","OriginalFileName","Hashes","Signed","Signature","SignatureStatus") ' \
" VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT" \
' ("Filename") DO UPDATE SET' \
' "FileVersion" = EXCLUDED."FileVersion", "Description" = EXCLUDED."Description",' \
' "Product" = EXCLUDED."Product", "Company" = EXCLUDED."Company",' \
'"OriginalFileName" = EXCLUDED."OriginalFileName","Hashes" = EXCLUDED."Hashes"' \
',"Signed" = EXCLUDED."Signed","Signature" = EXCLUDED."Signature",' \
'"SignatureStatus" = EXCLUDED."SignatureStatus";'.format(
"f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(),
event["Event"]["EventData"]["FileVersion"],
event["Event"]["EventData"]["Description"],
event["Event"]["EventData"]["Product"],
event["Event"]["EventData"]["Company"],
event["Event"]["EventData"]["OriginalFileName"],
event["Event"]["EventData"]["Hashes"],
event["Event"]["EventData"]["Signed"],
event["Event"]["EventData"]["Signature"],
event["Event"]["EventData"]["SignatureStatus"])
cursor.execute(query_file)
except Exception as e:
logger.error("Error query_file 7: " + str(e) + " Event: " + str(query_file))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"LoadImage",
"f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(),
(event["Event"]["EventData"]["Image"]).lower())
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 7: " + str(e) + " Event: " + str(query_action))
# Create Remote Thread
if event["Event"]["System"]["EventID"] == 8:
try: # Source Process
query_sprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image")' \
" VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;"\
.format((event["Event"]["EventData"]["SourceImage"]).lower(),
event["Event"]["EventData"]["SourceProcessId"],
(event["Event"]["EventData"]["SourceImage"]).lower())
cursor.execute(query_sprocess)
except Exception as e:
logger.error("Error query_sprocess 8: " + str(e) + " Event: " + str(query_sprocess))
try: # Target Process
query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["TargetProcessId"],
(event["Event"]["EventData"]["TargetImage"]).lower())
cursor.execute(query_tprocess)
except Exception as e:
logger.error("Error query_tprocess 8: " + str(e) + " Event: " + str(query_tprocess))
try: # Thread
thread_key = str(event["Event"]["EventData"]["TargetProcessGuid"]) + ":" + str(event["Event"]["EventData"]["NewThreadId"])
query_thread = 'INSERT INTO public."Threads" ("ThreadId","ThreadNId","ProcessGuid","StartAddress",' \
'"StartModule", "StartFunction")' \
" VALUES ('{}','{}','{}','{}','{}','{}') ON CONFLICT " \
'("ThreadId") DO UPDATE SET "StartAddress" = ' \
'EXCLUDED."StartAddress", "StartModule"' \
' = EXCLUDED."StartModule", "StartFunction"' \
' = EXCLUDED."StartFunction";'.format(
thread_key,
event["Event"]["EventData"]["NewThreadId"],
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["StartAddress"],
event["Event"]["EventData"]["StartModule"],
event["Event"]["EventData"]["StartFunction"])
cursor.execute(query_thread)
except Exception as e:
logger.error("Error query_thread 8: " + str(e) + " Event: " + str(query_thread))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"CreateRemoteThread",
(event["Event"]["EventData"]["SourceImage"]).lower(),
thread_key)
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 8: " + str(e) + " Event: " + str(query_action))
# Raw access read
if event["Event"]["System"]["EventID"] == 9:
logger.info("ToDo")
# Process Access
if event["Event"]["System"]["EventID"] == 10:
try:
query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["SourceImage"]).lower(),
event["Event"]["EventData"]["SourceProcessId"],
(event["Event"]["EventData"]["SourceImage"]).lower())
cursor.execute(query_pprocess)
except Exception as e:
logger.error("Error query_Pprocess 10: " + str(e) + " Event: " + str(query_pprocess))
try:
query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["TargetProcessId"],
(event["Event"]["EventData"]["TargetImage"]).lower())
cursor.execute(query_tprocess)
except Exception as e:
logger.error("Error query_tprocess 10: " + str(e) + " Event: " + str(query_tprocess))
try:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId",' \
'"ExtraInfo","ExtraInfo2")' \
" VALUES ('{}','{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"], "ProcessAccess",
(event["Event"]["EventData"]["SourceImage"]).lower(),
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["GrantedAccess"],
event["Event"]["EventData"]["CallTrace"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 10: " + str(e) + " Event: " + str(query_action))
# File create
if event["Event"]["System"]["EventID"] == 11: # Create File
try: # Process
insert_process(cursor, event)
except Exception as e:
print("Error query_sprocess 11: " + str(e) + " Event: " + str(event["Event"]))
try: # File
query_file = 'INSERT INTO public."Files" ("Filename","CreationUtcTime") ' \
"VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format(
"f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower(),
event["Event"]["EventData"]["CreationUtcTime"])
cursor.execute(query_file)
except Exception as e:
logger.error("Error query_file 11: " + str(e) + " Event: " + str(query_file))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"CreateFile",
(event["Event"]["EventData"]["Image"]).lower(),
"f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower())
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 11: " + str(e) + " Event: " + str(query_action))
# Registry Key Operation
if event["Event"]["System"]["EventID"] == 12 or event["Event"]["System"]["EventID"] == 13 \
or event["Event"]["System"]["EventID"] == 14:
if event["Event"]["EventData"]["TargetObject"]:
event["Event"]["EventData"]["TargetObject"] = str(
event["Event"]["EventData"]["TargetObject"]).replace("'", "\"")
try: # Process
insert_process(cursor, event)
except Exception as e:
logger.error("Error query_process 12-13-14: " + str(e) + " Event: " + str(event["Event"]))
try: # RegistryKey
if event["Event"]["System"]["EventID"] == 13:
if event["Event"]["EventData"]["Details"]:
event["Event"]["EventData"]["Details"] = str(
event["Event"]["EventData"]["Details"]).replace("'", "\"")
query_key = 'INSERT INTO public."RegistryKeys" ("Key","Details") ' \
"VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format(
event["Event"]["EventData"]["TargetObject"],
event["Event"]["EventData"]["Details"])
else:
query_key = 'INSERT INTO public."RegistryKeys" ("Key") ' \
"VALUES ('{}') ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["TargetObject"])
cursor.execute(query_key)
except Exception as e:
logger.error("Error query_key 12-13-14: " + str(e) + " Event: " + str(query_key))
try: # Action
if "SetValue" in event["Event"]["EventData"]["EventType"]:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId","ExtraInfo")' \
" VALUES ('{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"RegistryKey-" + event["Event"]["EventData"]["EventType"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["TargetObject"],
event["Event"]["EventData"]["Details"])
else:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"RegistryKey-" + event["Event"]["EventData"]["EventType"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["TargetObject"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 12-13-14: " + str(e) + " Event: " + str(query_action))
# File create stream hash
if event["Event"]["System"]["EventID"] == 15:
logger.info("ToDo")
# File create stream hash
if event["Event"]["System"]["EventID"] == 15:
logger.info("ToDo")
# Pipe event
if event["Event"]["System"]["EventID"] == 17 or event["Event"]["System"]["EventID"] == 18:
try:
insert_process(cursor, event)
except Exception as e:
print("Error insert_process 17-18: " + str(e) + " Event: " + str(event["Event"]))
try: # Pipe
if event["Event"]["EventData"]["PipeName"] not in pipes_inserted:
query_pipe = 'INSERT INTO public."Pipes" ("PipeName") ' \
"VALUES ('{}');".format(
event["Event"]["EventData"]["PipeName"])
cursor.execute(query_pipe)
pipes_inserted.append(event["Event"]["EventData"]["PipeName"])
except Exception as e:
logger.error("Error query_file 17-18: " + str(e) + " Event: " + str(query_pipe))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
event["Event"]["EventData"]["EventType"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["PipeName"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 17-18: " + str(e) + " Event: " + str(query_action))
# WMI event
if event["Event"]["System"]["EventID"] == 19 or event["Event"]["System"]["EventID"] == 20 \
or event["Event"]["System"]["EventID"] == 21:
logger.info("ToDo")
# DNS
if event["Event"]["System"]["EventID"] == 22:
try:
insert_process(cursor, event)
except Exception as e:
logger.error("Error insert_process 22: " + str(e) + " Event: " + str(event["Event"]))
try: # Query
query_dnsquery = 'INSERT INTO public."DNSQuery" ("QueryName") ' \
"VALUES ('{}') ON CONFLICT DO NOTHING;".format(
event["Event"]["EventData"]["QueryName"])
cursor.execute(query_dnsquery)
except Exception as e:
logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsquery))
try: # Resolution
query_dnsresolution = 'INSERT INTO public."DNSResolution" ("UtcTime","QueryName","QueryStatus","QueryResults") ' \
"VALUES ('{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format(
event["Event"]["EventData"]["UtcTime"],
event["Event"]["EventData"]["QueryName"],
event["Event"]["EventData"]["QueryStatus"],
event["Event"]["EventData"]["QueryResults"])
cursor.execute(query_dnsresolution)
except Exception as e:
logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsresolution))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}') ;".format(
event["Event"]["EventData"]["UtcTime"],
"DnsRequest",
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["QueryName"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 22: " + str(e) + " Event: " + str(query_action))
cursor.close()
end = time.time()
logger.info("Time to process file %s seconds ---" % (end - start))
def parseevtx(infile, outdir, coldelimiter, jsonfile):
outfilenametmp = os.path.basename(infile) + ".csv"
if jsonfile == 1:
jsonoutfilenametmp = os.path.basename(infile) + ".json"
drive, outfiledir = os.path.splitdrive(os.path.dirname(infile))
#Make Dir
harddir = outdir + os.sep + outfiledir
outfilename = harddir + os.sep + outfilenametmp
os.makedirs(harddir, exist_ok=True)
print(f'CVS Out Info : {outfilename}')
if jsonfile == 1:
jsonoutfilename = harddir + os.sep + jsonoutfilenametmp
print(f'JSON Out Info: {jsonoutfilename}')
jsonoutfile = open(jsonoutfilename, 'w')
# #1 #2 #3 #4 #5 #6 #7 #8 #9 #10 #11
header = "'EventID'|'ComputerName'|'TimeCreated'|'Security SID'|'UserID'|'Address'|'Script/IP:Port'|'Payload/Path/Correlation Activity ID'|'Event'|'Channel (EVTX Log)'|'SRC File'\n"
#create file
outfile = open(outfilename, 'w')
outfile.write(
"Please note that depending on the CSV deliminater used, the fields may not line up.\n"
)
outfile.write(header)
parser = PyEvtxParser(infile)
print(f'Working on: {infile}')
for record in parser.records_json():
subrec = json.loads(record["data"])
lineout = ""
try:
eventid = subrec['Event']['System']['EventID'] #['#text']
except:
eventid = subrec['Event']['System']['EventID']['#text']
channel = subrec['Event']['System']['Channel']
computer = subrec['Event']['System']['Computer']
systemtime = subrec['Event']['System']['TimeCreated']['#attributes'][
'SystemTime']
#print(subrec['Event']['UserData']['EventXML'].keys())
if jsonfile == 1:
if channel == "Windows PowerShell" or channel == "Microsoft-Windows-WinRM/Operational" or channel == "Microsoft-Windows-PowerShell/Operational" or channel == "Security" or channel == "System":
jsonoutfile.write(printjson(subrec))
jsonoutfile.write("\n")
#print(f'EID: {eventid}')
#--------------------------------------------------------------------------------------------------------------------------------
#Windows PowerShell.evtx
if eventid == 400 or eventid == 403 or eventid == 600:
if channel == "Windows PowerShell":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['Data']['#text'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
#Microsoft-Windows-WinRM/Operational.evtx
if eventid == 6:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Connecting remotely to: " + subrec[
'Event']['EventData'][
'connection'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 81:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Operation Name: " + subrec['Event'][
'EventData']['operationName'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 82:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Operation: " + subrec['Event'][
'EventData'][
'operation'] + " -- Resource URI: " + subrec[
'Event']['EventData'][
'resourceURI'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 134:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Operation: " + subrec['Event'][
'EventData']['operationName'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
# Need an example log related to powershell; now it prints out all 142 events
if eventid == 142:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'OperatinName: " + subrec['Event'][
'EventData'][
'operationName'] + " -- Error Code :'" + str(
subrec['Event']['EventData']
['errorCode']) + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 169:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "'" + subrec['Event']['EventData'][
'username'] + "'" + coldelimiter
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Authentication Mechanism: " + subrec[
'Event']['EventData'][
'authenticationMechanism'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
#Microsoft-Windows-PowerShell/Operational.evtx
if eventid == 4100:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['ContextInfo'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "'" + subrec['Event']['EventData'][
'Payload'] + "'" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 4103:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec) ScriptBlockText
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['ContextInfo'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "'" + subrec['Event']['EventData'][
'Path'] + "'" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 4104:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['ScriptBlockText'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "'" + subrec['Event']['EventData'][
'Path'] + "'" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 40961:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
#Security
if eventid == 4688:
if channel == "Security":
try:
if re.search(
'powershell',
subrec['Event']['EventData']['NewProcessName'],
re.IGNORECASE):
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['EventData'][
'SubjectDomainName'] + "\\" + subrec['Event'][
'EventData']['SubjectUserName'] + " (" + subrec[
'Event']['EventData'][
'SubjectUserSid'] + ")'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
try:
tmpstr = subrec['Event']['EventData'][
'CommandLine']
try:
tmpppn = subrec['Event']['EventData'][
'ParentProcessName']
except:
tmpppn = ''
if tmpstr:
lineout = lineout + "'" + tmpppn + " -> " + subrec[
'Event']['EventData'][
'CommandLine'] + "'" + coldelimiter #7
else:
lineout = lineout + "'" + tmpppn + " -> " + subrec[
'Event']['EventData'][
'NewProcessName'] + "'" + coldelimiter #7
except:
lineout = lineout + "'" + tmpppn + " -> " + subrec[
'Event']['EventData'][
'NewProcessName'] + "'" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
if eventid == 7030 or eventid == 7040 or eventid == 7045:
if channel == "System":
try:
printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
print("end")
outfile.close()
if jsonfile == 1:
jsonoutfile.close()
class Evtx2es(object):
def __init__(self, filepath: str):
self.path = Path(filepath).resolve()
self.parser = PyEvtxParser(self.path.open(mode="rb"))
def format_record(self, record: dict) -> dict:
record["data"] = orjson.loads(record.get("data"))
eventid_field = record.get("data", {}).get("Event",
{}).get("System",
{}).get("EventID")
if type(eventid_field) is dict:
record["data"]["Event"]["System"]["EventID"] = eventid_field.get(
"#text")
try:
status = record.get("data").get("Event").get("EventData").get(
"Status")
record["data"]["Event"]["EventData"]["Status"] = None
except Exception:
pass
# Convert data according to ECS (sort of)
# First copy system fields
record["winlog"] = {
"channel":
record["data"]["Event"]["System"]["Channel"],
"computer_name":
record["data"]["Event"]["System"]["Computer"],
"event_id":
record["data"]["Event"]["System"]["EventID"],
"opcode":
record["data"]["Event"]["System"].get("Opcode"),
"provider_guid":
record["data"]["Event"]["System"]["Provider"]["#attributes"].get(
"Guid"),
"provider_name":
record["data"]["Event"]["System"]["Provider"]["#attributes"]
["Name"],
"record_id":
record["data"]["Event"]["System"]["EventRecordID"],
"task":
record["data"]["Event"]["System"]["Task"],
"version":
record["data"]["Event"]["System"].get("Version"),
}
try:
record["winlog"]["process"] = {
"pid":
record["data"]["Event"]["System"]["Execution"]["#attributes"]
["ProcessID"],
"thread_id":
record["data"]["Event"]["System"]["Execution"]["#attributes"]
["ThreadID"],
}
except KeyError:
pass
except TypeError:
pass
try:
record["userdata"] = {
"address":
record["data"]["Event"]["UserData"]["EventXML"]["Address"],
"sessionid":
record["data"]["Event"]["UserData"]["EventXML"]["SessionID"],
"user":
record["data"]["Event"]["UserData"]["EventXML"]["User"],
}
except KeyError:
pass
except TypeError:
pass
record.update({
"log": {
"file": {
"name": str(self.path)
}
},
"event": {
"code":
record["winlog"]["event_id"],
"created":
record["data"]["Event"]["System"]["TimeCreated"]["#attributes"]
["SystemTime"],
},
})
record["@timestamp"] = record["event"]["created"]
# Move event attributes to ECS location
record["winlog"]["event_data"] = record["data"]["Event"].get(
"EventData", dict())
del record["data"]
if (record["winlog"]["event_data"] is None
or len(record["winlog"]["event_data"])
== 0): # remove event_data fields if empty
del record["winlog"]["event_data"]
else:
if record["winlog"]["event_data"]:
for k, v in record["winlog"]["event_data"].items():
# Normalize some known problematic fields with values switching between integers and strings with hexadecimal notation to integers
if k in ("ProcessId") and type(v) == str:
if v.startswith("0x"):
record["winlog"]["event_data"][k] = int(v, 16)
else:
try:
record["winlog"]["event_data"][k] = int(v)
except ValueError:
record["winlog"]["event_data"][k] = 0
# Maximum limit of numeric values in Elasticsearch
if type(v) is int:
if v < -(2**63):
record["winlog"]["event_data"][k] = -(2**63)
elif v > 2**63 - 1:
record["winlog"]["event_data"][k] = 2**63 - 1
return record
def gen_records(self, size: int) -> Generator:
"""A generator that reads records from an Evtx file and generates a dict for each record.
Args:
size (int): Buffer size.
Yields:
Generator: Yields List[dict].
"""
buffer: List[dict] = list()
# generates records
for record in self.parser.records_json():
formatted_record: dict = self.format_record(record)
buffer.append(formatted_record)
if len(buffer) >= size:
yield buffer
buffer.clear()
else:
yield buffer
