def xml_records(filename):
if OPTIONS.alternate:
with Evtx(filename) as evtx:
try:
for xml, record in evtx_file_xml_view(evtx.get_file_header()):
try:
yield ef.to_lxml(xml), None
except etree.XMLSyntaxError as e:
yield xml, e
except BinaryParser.OverrunBufferException as e:
logging.error("Overrun Buffer Exception!")
yield None, e
except BinaryParser.ParseException as e:
logging.error("Parse Exception!")
yield None, e
except Exception as e: # UnicodeDecodeError, AttributeError
logging.error(e)
yield None, e
else:
parser = PyEvtxParser(filename)
try:
for record in parser.records():
try:
yield ef.to_lxml(record['data']), None
except etree.XMLSyntaxError as e:
yield record['data'], e
except Exception as e: # UnicodeDecodeError, AttributeError, RuntimeError
logging.error(e)
yield None, e
def _read_evt_logs(func, logs_file: str, result_file: str,
column_names: List[str]):
global_start = time.time()
start = time.time()
_row_values = dict()
if os.path.isfile(result_file):
os.remove(result_file)
with open(result_file, 'w', encoding='utf-8', newline='') as file:
writer = csv.DictWriter(file, column_names)
writer.writeheader()
log = PyEvtxParser(logs_file)
for index, record in enumerate(log.records()):
_row_values.clear()
_row_values = dict.fromkeys(column_names, 0)
log_data = record['data']
func(log_data, _row_values)
if IS_DEBUG:
if index == 1000:
break
with open(result_file, 'a', encoding='utf-8', newline='') as file:
writer = csv.DictWriter(file, column_names)
writer.writerow(_row_values)
if index % 10000 == 0:
stop = time.time()
print(f"{index}: {stop - start} сек.")
start = time.time()
print(f"затраченно времени: {time.time() - global_start} сек.")
def parse(self):
parser = PyEvtxParser(self.eventfile)
for record in parser.records_json():
rec = json.loads(record['data'])['Event']
data = {}
# Common fields
data['event.created'] = record.get(
'timestamp',
rec['System']['TimeCreated']['#attributes']['SystemTime'])
if isinstance(rec['System']['EventID'], dict):
data['event.code'] = str(rec['System']['EventID']['#text'])
else:
data['event.code'] = str(rec['System']['EventID'])
data['event.provider'] = rec['System']['Provider']['#attributes'][
'Name']
data['event.dataset'] = rec['System']['Channel']
if 'Security' in rec['System']:
try:
data['user.id'] = rec['System']['Security']['#attributes'][
'UserID']
except Exception:
pass
try:
data['process.pid'] = rec['System']['Execution'][
'#attributes']['ProcessID']
data['process.thread.id'] = rec['System']['Execution'][
'#attributes']['ThreadID']
except Exception:
pass
# Events not defined in data_json
if not data['event.code'] in self.data_json.keys(
) or data['event.provider'] != self.data_json[
data['event.code']]['provider']:
# EventData, UserData are just reproduced as dictionaries
if 'EventData' in rec:
data['EventData'] = rec['EventData']
if 'UserData' in rec:
data['UserData'] = rec['UserData']
yield data
continue
# Selected events
data['description'] = self.data_json[
data['event.code']]["description"]
for field in ['category', 'type', 'action']:
if field in self.data_json[data['event.code']]:
data['event.{}'.format(field)] = self.data_json[
data['event.code']][field]
if 'path' not in self.data_json[data['event.code']].keys():
yield data
continue
# Extra fields
for x, item in self.data_json[data['event.code']]['path'].items():
self.get_xpath_data(x, item, rec, data)
yield data
def test_it_works_with_records(small_sample):
parser = PyEvtxParser(small_sample)
records = list(parser.records())
assert len(records) == 7
assert records[0]['event_record_id'] == 7
assert records[0]['timestamp'].endswith('UTC')
assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_fails_nicely_on_close_files(small_sample):
with open(small_sample, "rb") as o:
parser = PyEvtxParser(o)
with pytest.raises(OSError) as e:
records = list(parser.records())
assert "closed file" in e.value.args[1]
def test_it_works_with_json(small_sample):
parser = PyEvtxParser(small_sample)
records = list(parser.records_json())
assert len(records) == 7
assert records[0]['event_record_id'] == 7
assert records[0]['timestamp'].endswith('UTC')
assert json.loads(records[0]['data'])['Event']['System']['EventID'] == 4673
def parserecord(filename,
num_items,
logBufferSize,
index,
nodes,
debug,
support_queue,
token=""):
count = 0
JSONevents = ""
logBufferLength = 0
count_postedrecord = 0
parser = PyEvtxParser(filename)
for record in parser.records_json():
count = count + 1
logBufferLength = logBufferLength + 1
record = json.loads(record['data'])
if isinstance(record['Event']['System']['EventID'], int):
eventid = record['Event']['System']['EventID']
record['Event']['System']['EventID'] = {}
record['Event']['System']['EventID']['#text'] = eventid
# Same thing with their Data field
try:
if isinstance(record['Event']['EventData']['Data'], str):
tmp = record['Event']['EventData']['Data']
record['Event']['EventData']['Data'] = {}
record['Event']['EventData']['Data']['#text'] = tmp
except KeyError:
pass
except TypeError:
pass
JSONevents = JSONevents + '{"index": {}}\n'
JSONevents = JSONevents + json.dumps(record) + "\n"
# Dump log buffer when full
if logBufferLength >= int(logBufferSize):
count_postedrecord = count_postedrecord + logBufferSize
dump_batch(JSONevents, index, nodes, count_postedrecord, num_items,
debug, support_queue, token)
JSONevents = ""
logBufferLength = 0
if logBufferLength > 0:
count_postedrecord = count_postedrecord + logBufferLength
dump_batch(JSONevents, index, nodes, count_postedrecord, num_items,
debug, support_queue, token)
logBufferLength = 0
JSONevents = ""
return count_postedrecord
def test_it_supports_various_num_threads(small_sample):
with open(small_sample, "rb") as o:
parser = PyEvtxParser(o, number_of_threads=1)
records = list(parser.records())
assert len(records) == 7
assert records[0]['event_record_id'] == 7
assert records[0]['timestamp'].endswith('UTC')
assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_supports_various_ascii_codecs(small_sample):
with open(small_sample, "rb") as o:
parser = PyEvtxParser(o, ansi_codec="ascii")
records = list(parser.records())
assert len(records) == 7
assert records[0]['event_record_id'] == 7
assert records[0]['timestamp'].endswith('UTC')
assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_works_on_file_backed_object(small_sample):
with open(small_sample, "rb") as o:
parser = PyEvtxParser(o)
records = list(parser.records())
assert len(records) == 7
assert records[0]['event_record_id'] == 7
assert records[0]['timestamp'].endswith('UTC')
assert '<EventID>4673</EventID>' in records[0]['data']
def test_it_works_on_io_object(small_sample):
with open(small_sample, "rb") as o:
r = o.read()
parser = PyEvtxParser(io.BytesIO(r))
records = list(parser.records())
assert len(records) == 7
assert records[0]['event_record_id'] == 7
assert records[0]['timestamp'].endswith('UTC')
assert '<EventID>4673</EventID>' in records[0]['data']
def run(self, target_file: TargetFile,
result_of_previos_modules: dict) -> dict:
events = []
self._filter_ids = self._get_filter_ids_from_params()
parser = PyEvtxParser(target_file.get_path())
for record in parser.records():
try:
parsed = xmltodict.parse(record["data"])
event = self._get_info(parsed, target_file)
if event is not None:
events.append(event)
except:
pass
piped = utils.pipe_to_another_output(self._params, events)
return {"n_events": len(events)} if piped else {"events": events}
def runUsingBindings(self, file):
"""
Convert EVTX to JSON using evtx_dump bindings (slower)
Drop resulting JSON files in a tmp folder.
"""
try:
filepath = Path(file)
filename = filepath.name
parser = PyEvtxParser(str(filepath))
with open(
f"{self.tmpDir}/{str(filename)}-{self.randString()}.json",
"w") as f:
for record in parser.records_json():
f.write(f'{json.dumps(json.loads(record["data"]))}\n')
except Exception as e:
self.logger.error(f"{Fore.RED} [-] {e}")
def MatchLogFile(self, filePath):
parser = PyEvtxParser(filePath)
if self.logger:
self.logger.info(
f"Statrted processing the Event log '{filePath}'.")
for record in parser.records_json():
try:
data = json.loads(record["data"])
event = Event(data)
self.matchAll(event)
except Exception as e:
if self.logger:
self.logger.error(e, exc_info=True)
if self.logger:
self.logger.info(
f"Finished processing the Event log '{filePath}'.")
def asyncGenLogs(file):
EVTx = {}
json_data_dir = createDir('web\json_data')
print('EVTxFILE:', file)
fname = filename(file)
_df = pd.DataFrame()
parser = PyEvtxParser(file)
i = 0 # Register counter
for record in parser.records_json():
data = json.loads(record['data'])['Event']
readXmlObj(data, EVTx)
try:
df = pd.DataFrame.from_dict(EVTx).fillna(0)
_df = _df.append(df)
i += 1
except Exception as e:
pass
EVTx = {}
if i > (REG_COUNT_LIMIT - 1):
break
if not _df.empty:
if REGEX_PATERN:
try:
_df = _df[_df.apply(lambda x: x.astype(str).str.contains(
REGEX_PATERN, case=False, na=False)).any(axis=1)]
except Exception as e:
# error(e) #verbose
pass
_df = _df.set_index('EventRecordID').sort_values(by=['EventRecordID'],
ascending=False)
_df.to_json(f'{json_data_dir}\\{md5(fname)}.json', orient="table")
r, c = _df.shape
eel.feedback(f'Lines: {r}, Columns: {c}', md5(file))
else:
pd.DataFrame(['Empty'],
columns=['File'
]).to_json(f'{json_data_dir}\\{md5(fname)}.json',
index=False,
orient="table")
eel.feedback(f'Empty', md5(file))
def Evtx_hunt(files, str_regex, input_timzone, output):
for file in files:
print("Analyzing " + file)
try:
parser = PyEvtxParser(file)
except:
print("Issue analyzing " + file +
"\nplease check if its not corrupted")
continue
try:
rex = re.compile(str_regex, re.IGNORECASE)
for record in parser.records():
EventID = EventID_rex.findall(record['data'])
if len(EventID) > 0:
Computer = Computer_rex.findall(record['data'])
Channel = Channel_rex.findall(record['data'])
if len(Channel) > 0:
channel = Channel[0]
else:
channel = " "
#print(record['data'])
# if record['data'].lower().find(str_regex.lower())>-1:
if rex.findall(record['data']):
#print("EventID : "+EventID[0]+" , Data : "+record['data'])
Hunting_events[0]['timestamp'].append(
datetime.timestamp(
isoparse(
parse(record["timestamp"]).astimezone(
input_timzone).isoformat())))
Hunting_events[0]['Date and Time'].append(
parse(record["timestamp"]).astimezone(
input_timzone).isoformat())
Hunting_events[0]['Channel'].append(channel)
Hunting_events[0]['Event ID'].append(EventID[0])
Hunting_events[0]['Computer'].append(Computer[0])
Hunting_events[0]['Original Event Log'].append(
str(record['data']).replace("\r", " ").replace(
"\n", " "))
except Exception as e:
print("issue searching log : " + record['data'] + "\n Error : " +
print(e))
hunt_report(output)
class Evtx2es(object):
def __init__(self, input_path: Path) -> None:
self.path = input_path
self.parser = PyEvtxParser(self.path.open(mode="rb"))
def gen_records(self, shift: Union[str, datetime], multiprocess: bool,
chunk_size: int) -> Generator:
"""Generates the formatted Eventlog records chunks.
Args:
multiprocess (bool): Flag to run multiprocessing.
chunk_size (int): Size of the chunk to be processed for each process.
Yields:
Generator: Yields List[dict].
"""
gen_path = iter(lambda: str(self.path), None)
gen_shift = iter(lambda: shift, None)
if multiprocess:
with Pool(cpu_count()) as pool:
results = pool.starmap_async(
process_by_chunk,
zip(
generate_chunks(chunk_size,
self.parser.records_json()),
gen_path,
gen_shift,
))
yield list(chain.from_iterable(results.get(timeout=None)))
else:
buffer: List[List[dict]] = list()
for records in generate_chunks(chunk_size,
self.parser.records_json()):
if chunk_size <= len(buffer):
yield list(chain.from_iterable(buffer))
buffer.clear()
else:
buffer.append(
process_by_chunk(records, gen_path, gen_shift))
else:
yield list(chain.from_iterable(buffer))
def load_events(log_file_name):
"""
Opens Sysmon logs as a readable file, and turns the events into dictionaries
:param log_file_name: Sysmon evtx or xml log to be opened
:return: dict of the Sysmon event log
"""
try:
with open(log_file_name, "r", encoding='utf-8', errors='ignore') as fp:
magic = fp.read(7)
fp.seek(0)
if magic == 'ElfFile':
# log is evtx type
parser = PyEvtxParser(log_file_name)
dictrecords = [json.loads(rec['data']) for rec in parser.records_json()]
return dictrecords, 'evtx'
elif magic == '<Events':
return xmltodict.parse(fp.read()), 'xml'
else:
raise Exception('Invalid File magic')
except ExpatError:
raise KeyError("Error: Format error in the Event log file")
def read_evt_logs(func,
logs_file: str = 'data/Security.evtx',
result_file: str = 'data/column_names_utf_8'):
global_start = time.time()
start = time.time()
stop = 0
log = PyEvtxParser(logs_file)
for index, record in enumerate(log.records()):
# if index > 0:
# break
log_data = record['data']
func(log_data)
if index % 10000 == 0:
stop = time.time()
print(f"{index}: {stop - start} сек.")
start = time.time()
try:
with open(result_file, 'w', encoding='utf-8') as file:
json.dump(column_names, file)
except Exception as ex:
print(ex)
print(f"затраченно времени: {time.time() - global_start} сек.")
class EvtxViewModel(QAbstractItemModel):
def __init__(self, filename):
super(QAbstractItemModel, self).__init__()
self.__filename = filename
self.__chunks = 0
self.__columns = [
('TimeCreated', lambda r: r.timestamp),
('Provider', lambda r: r.Provider),
('EventID', lambda r: r.EventID),
('EventData', lambda r: r.Level)
]
for idx in range(0, len(self.__columns)):
self.setHeaderData(idx, Qt.Horizontal, self.__columns[0][0], Qt.DisplayRole)
self.__records = dict()
self.__record_ids = list()
self.load_data()
def load_data(self):
self.__parser = PyEvtxParser(self.__filename)
for record in self.__parser.records():
self.__records[record["event_record_id"]] = EventRecord(record)
self.__record_ids.append(record["event_record_id"])
self.__record_ids.sort()
def rowCount(self, parent):
return len(self.__record_ids)
def columnCount(self, parent):
return len(self.__columns)
def index(self, row: int, column: int, parent: QModelIndex = ...) -> QModelIndex:
return self.createIndex(row, column)
def data(self, index: QModelIndex, role: int = ...) -> typing.Any:
if role == Qt.DisplayRole:
row = index.row()
record_id = self.__record_ids[row]
record = self.__records[record_id]
column = self.__columns[index.column()]
return column[1](record)
else:
return QVariant()
def flags(self, index: QModelIndex) -> Qt.ItemFlags:
return Qt.ItemIsEnabled | Qt.ItemNeverHasChildren
def parent(self, qmodelindex=None):
return QModelIndex()
def test_it_returns_error_when_iterating_twice(small_sample):
parser = PyEvtxParser(small_sample)
_ = list(parser.records())
with pytest.raises(RuntimeError):
parser.records()
def test_it_returns_error_when_using_next_on_parser(small_sample):
parser = PyEvtxParser(small_sample)
with pytest.raises(NotImplementedError):
next(parser)
def parser_simple(path, opath=None):
import time
cursor = sql_connection()
sql_initialitation(cursor)
parser = PyEvtxParser(path)
connections_key = 100000000
thread_key = 0
files_inserted = []
full_process_inserted = []
pipes_inserted = []
threads_inserted = []
start = time.time()
for record in parser.records_json():
event = json.loads(record['data'])
# Process Creation
if event["Event"]["System"]["EventID"] == 1:
try:
if event["Event"]["EventData"]["ProcessGuid"] not in full_process_inserted:
#print("Full inserted")
query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image","IntegrityLevel",' \
'"TerminalSessionId", "User")' \
" VALUES ('{}','{}','{}','{}','{}', '{}')" \
' ON CONFLICT ("ProcessGuid") DO UPDATE SET "IntegrityLevel" = ' \
'EXCLUDED."IntegrityLevel", "TerminalSessionId"' \
' = EXCLUDED."TerminalSessionId", "User" = EXCLUDED."User";'.format(
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["ProcessId"],
event["Event"]["EventData"]["Image"],
event["Event"]["EventData"]["IntegrityLevel"],
event["Event"]["EventData"]["TerminalSessionId"],
event["Event"]["EventData"]["User"])
cursor.execute(query_process)
full_process_inserted.append(event["Event"]["EventData"]["ProcessGuid"])
except Exception as e:
logger.error("Error query_process 1: " + str(e) + " Event: " + str(event["Event"]))
try:
if event["Event"]["EventData"]["Image"] not in files_inserted:
if "OriginalFileName" in event["Event"]["EventData"]:
original = ",'" + str(event["Event"]["EventData"]["OriginalFileName"]) + "'"
key = ',"OriginalFileName"'
else:
original = ""
key = ""
except Exception as e:
logger.error("Error query_file 1: " + str(e) + " Event: " + str(event["Event"]))
try:
query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["ParentImage"]).lower(),
event["Event"]["EventData"]["ParentProcessId"],
event["Event"]["EventData"]["ParentImage"])
cursor.execute(query_pprocess)
except Exception as e:
logger.error("Error query_pprocess_exist 1: " + str(e) + " Event: " + str(query_pprocess))
try:
if event["Event"]["EventData"]["CommandLine"]:
event["Event"]["EventData"]["CommandLine"] = str(
event["Event"]["EventData"]["CommandLine"]).replace("'", "\"")
if event["Event"]["EventData"]["CurrentDirectory"]:
event["Event"]["EventData"]["CurrentDirectory"] = str(
event["Event"]["EventData"]["CurrentDirectory"]).replace("'", "\"")
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \
'"ExtraInfo","ExtraInfo2")' \
" VALUES ('{}','{}','{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"], "CreateProcess",
(event["Event"]["EventData"]["ParentImage"]).lower(),
event["Event"]["EventData"]["LogonGuid"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["CommandLine"],
event["Event"]["EventData"]["CurrentDirectory"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 1: " + str(e) + " Event: " + str(query_action))
try:
query_user = '******' \
" ('{}','{}','{}',{}) ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["LogonGuid"],
event["Event"]["EventData"]["User"],
event["Event"]["EventData"]["LogonId"],
event["Event"]["EventData"]["TerminalSessionId"])
cursor.execute(query_user)
except Exception as e:
logger.error("Error 1: " + str(e) + " Event: " + str(query_user))
# File creation time changed
if event["Event"]["System"]["EventID"] == 2:
logger.info("ToDo")
# Network connection
if event["Event"]["System"]["EventID"] == 3:
try:
connections_key = str(event["Event"]["EventData"]["SourceIp"])+str(event["Event"]["EventData"]["DestinationIp"])
query_connection = 'INSERT INTO public."Connections" ("ConnectionId","Protocol","SourceIp","SourceHostname",' \
'"SourcePort","DestinationIsIpv6","DestinationIp","DestinationHostname","DestinationPort") ' \
"VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format(
connections_key, event["Event"]["EventData"]["Protocol"],
event["Event"]["EventData"]["SourceIp"],
event["Event"]["EventData"]["SourceHostname"],
event["Event"]["EventData"]["SourcePort"],
event["Event"]["EventData"]["DestinationIsIpv6"],
event["Event"]["EventData"]["DestinationIp"],
event["Event"]["EventData"]["DestinationHostname"],
event["Event"]["EventData"]["DestinationPort"])
cursor.execute(query_connection)
except Exception as e:
logger.error("Error query_connection 3: " + str(e) + " Event: " + str(query_connection))
try:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","LogonGuid","DestinationId",' \
'"ExtraInfo")' \
" VALUES ('{}','{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"], "CreateConnection",
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["User"],
connections_key,
event["Event"]["EventData"]["Initiated"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 3: " + str(e) + " Event: " + str(query_action))
try:
insert_process(cursor, event)
except Exception as e:
logger.error("Error query_pprocess_exist 3: " + str(e) + " Event: " + str(event["Event"]))
# Process Terminated
if event["Event"]["System"]["EventID"] == 5:
try: # Destination Process
query_process = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["ProcessId"],
event["Event"]["EventData"]["Image"])
cursor.execute(query_process)
except Exception as e:
logger.error("Error query_sprocess 5: " + str(e) + " Event: " + str(query_process))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"ProcessTerminated",
(event["Event"]["EventData"]["Image"]).lower(),
(event["Event"]["EventData"]["Image"]).lower())
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 5: " + str(e) + " Event: " + str(query_action))
# Kernel driver loaded
if event["Event"]["System"]["EventID"] == 6:
logger.info("ToDo")
# Image loaded
if event["Event"]["System"]["EventID"] == 7:
try: # Process
insert_process(cursor, event)
except Exception as e:
logger.error("Error query_sprocess 7: " + str(e) + " Event: " + str(query_sprocess))
try: # File
if "Description" not in event["Event"]["EventData"]:
event["Event"]["EventData"]["Description"] = ""
if "OriginalFileName" not in event["Event"]["EventData"]:
event["Event"]["EventData"]["OriginalFileName"] = ""
query_file = 'INSERT INTO public."Files" ("Filename","FileVersion","Description","Product","Company","OriginalFileName","Hashes","Signed","Signature","SignatureStatus") ' \
" VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}') ON CONFLICT" \
' ("Filename") DO UPDATE SET' \
' "FileVersion" = EXCLUDED."FileVersion", "Description" = EXCLUDED."Description",' \
' "Product" = EXCLUDED."Product", "Company" = EXCLUDED."Company",' \
'"OriginalFileName" = EXCLUDED."OriginalFileName","Hashes" = EXCLUDED."Hashes"' \
',"Signed" = EXCLUDED."Signed","Signature" = EXCLUDED."Signature",' \
'"SignatureStatus" = EXCLUDED."SignatureStatus";'.format(
"f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(),
event["Event"]["EventData"]["FileVersion"],
event["Event"]["EventData"]["Description"],
event["Event"]["EventData"]["Product"],
event["Event"]["EventData"]["Company"],
event["Event"]["EventData"]["OriginalFileName"],
event["Event"]["EventData"]["Hashes"],
event["Event"]["EventData"]["Signed"],
event["Event"]["EventData"]["Signature"],
event["Event"]["EventData"]["SignatureStatus"])
cursor.execute(query_file)
except Exception as e:
logger.error("Error query_file 7: " + str(e) + " Event: " + str(query_file))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"LoadImage",
"f:" + str(event["Event"]["EventData"]["ImageLoaded"]).lower(),
(event["Event"]["EventData"]["Image"]).lower())
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 7: " + str(e) + " Event: " + str(query_action))
# Create Remote Thread
if event["Event"]["System"]["EventID"] == 8:
try: # Source Process
query_sprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image")' \
" VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;"\
.format((event["Event"]["EventData"]["SourceImage"]).lower(),
event["Event"]["EventData"]["SourceProcessId"],
(event["Event"]["EventData"]["SourceImage"]).lower())
cursor.execute(query_sprocess)
except Exception as e:
logger.error("Error query_sprocess 8: " + str(e) + " Event: " + str(query_sprocess))
try: # Target Process
query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["TargetProcessId"],
(event["Event"]["EventData"]["TargetImage"]).lower())
cursor.execute(query_tprocess)
except Exception as e:
logger.error("Error query_tprocess 8: " + str(e) + " Event: " + str(query_tprocess))
try: # Thread
thread_key = str(event["Event"]["EventData"]["TargetProcessGuid"]) + ":" + str(event["Event"]["EventData"]["NewThreadId"])
query_thread = 'INSERT INTO public."Threads" ("ThreadId","ThreadNId","ProcessGuid","StartAddress",' \
'"StartModule", "StartFunction")' \
" VALUES ('{}','{}','{}','{}','{}','{}') ON CONFLICT " \
'("ThreadId") DO UPDATE SET "StartAddress" = ' \
'EXCLUDED."StartAddress", "StartModule"' \
' = EXCLUDED."StartModule", "StartFunction"' \
' = EXCLUDED."StartFunction";'.format(
thread_key,
event["Event"]["EventData"]["NewThreadId"],
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["StartAddress"],
event["Event"]["EventData"]["StartModule"],
event["Event"]["EventData"]["StartFunction"])
cursor.execute(query_thread)
except Exception as e:
logger.error("Error query_thread 8: " + str(e) + " Event: " + str(query_thread))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"CreateRemoteThread",
(event["Event"]["EventData"]["SourceImage"]).lower(),
thread_key)
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 8: " + str(e) + " Event: " + str(query_action))
# Raw access read
if event["Event"]["System"]["EventID"] == 9:
logger.info("ToDo")
# Process Access
if event["Event"]["System"]["EventID"] == 10:
try:
query_pprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["SourceImage"]).lower(),
event["Event"]["EventData"]["SourceProcessId"],
(event["Event"]["EventData"]["SourceImage"]).lower())
cursor.execute(query_pprocess)
except Exception as e:
logger.error("Error query_Pprocess 10: " + str(e) + " Event: " + str(query_pprocess))
try:
query_tprocess = 'INSERT INTO public."Processes" ("ProcessGuid","ProcessId","Image") ' \
"VALUES ('{}',{},'{}') ON CONFLICT DO NOTHING;".format(
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["TargetProcessId"],
(event["Event"]["EventData"]["TargetImage"]).lower())
cursor.execute(query_tprocess)
except Exception as e:
logger.error("Error query_tprocess 10: " + str(e) + " Event: " + str(query_tprocess))
try:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId",' \
'"ExtraInfo","ExtraInfo2")' \
" VALUES ('{}','{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"], "ProcessAccess",
(event["Event"]["EventData"]["SourceImage"]).lower(),
(event["Event"]["EventData"]["TargetImage"]).lower(),
event["Event"]["EventData"]["GrantedAccess"],
event["Event"]["EventData"]["CallTrace"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 10: " + str(e) + " Event: " + str(query_action))
# File create
if event["Event"]["System"]["EventID"] == 11: # Create File
try: # Process
insert_process(cursor, event)
except Exception as e:
print("Error query_sprocess 11: " + str(e) + " Event: " + str(event["Event"]))
try: # File
query_file = 'INSERT INTO public."Files" ("Filename","CreationUtcTime") ' \
"VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format(
"f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower(),
event["Event"]["EventData"]["CreationUtcTime"])
cursor.execute(query_file)
except Exception as e:
logger.error("Error query_file 11: " + str(e) + " Event: " + str(query_file))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"CreateFile",
(event["Event"]["EventData"]["Image"]).lower(),
"f:" + str(event["Event"]["EventData"]["TargetFilename"]).lower())
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 11: " + str(e) + " Event: " + str(query_action))
# Registry Key Operation
if event["Event"]["System"]["EventID"] == 12 or event["Event"]["System"]["EventID"] == 13 \
or event["Event"]["System"]["EventID"] == 14:
if event["Event"]["EventData"]["TargetObject"]:
event["Event"]["EventData"]["TargetObject"] = str(
event["Event"]["EventData"]["TargetObject"]).replace("'", "\"")
try: # Process
insert_process(cursor, event)
except Exception as e:
logger.error("Error query_process 12-13-14: " + str(e) + " Event: " + str(event["Event"]))
try: # RegistryKey
if event["Event"]["System"]["EventID"] == 13:
if event["Event"]["EventData"]["Details"]:
event["Event"]["EventData"]["Details"] = str(
event["Event"]["EventData"]["Details"]).replace("'", "\"")
query_key = 'INSERT INTO public."RegistryKeys" ("Key","Details") ' \
"VALUES ('{}','{}') ON CONFLICT DO NOTHING;".format(
event["Event"]["EventData"]["TargetObject"],
event["Event"]["EventData"]["Details"])
else:
query_key = 'INSERT INTO public."RegistryKeys" ("Key") ' \
"VALUES ('{}') ON CONFLICT DO NOTHING;".format(event["Event"]["EventData"]["TargetObject"])
cursor.execute(query_key)
except Exception as e:
logger.error("Error query_key 12-13-14: " + str(e) + " Event: " + str(query_key))
try: # Action
if "SetValue" in event["Event"]["EventData"]["EventType"]:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId","ExtraInfo")' \
" VALUES ('{}','{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"RegistryKey-" + event["Event"]["EventData"]["EventType"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["TargetObject"],
event["Event"]["EventData"]["Details"])
else:
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
"RegistryKey-" + event["Event"]["EventData"]["EventType"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["TargetObject"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 12-13-14: " + str(e) + " Event: " + str(query_action))
# File create stream hash
if event["Event"]["System"]["EventID"] == 15:
logger.info("ToDo")
# File create stream hash
if event["Event"]["System"]["EventID"] == 15:
logger.info("ToDo")
# Pipe event
if event["Event"]["System"]["EventID"] == 17 or event["Event"]["System"]["EventID"] == 18:
try:
insert_process(cursor, event)
except Exception as e:
print("Error insert_process 17-18: " + str(e) + " Event: " + str(event["Event"]))
try: # Pipe
if event["Event"]["EventData"]["PipeName"] not in pipes_inserted:
query_pipe = 'INSERT INTO public."Pipes" ("PipeName") ' \
"VALUES ('{}');".format(
event["Event"]["EventData"]["PipeName"])
cursor.execute(query_pipe)
pipes_inserted.append(event["Event"]["EventData"]["PipeName"])
except Exception as e:
logger.error("Error query_file 17-18: " + str(e) + " Event: " + str(query_pipe))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}');".format(
event["Event"]["EventData"]["UtcTime"],
event["Event"]["EventData"]["EventType"],
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["PipeName"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 17-18: " + str(e) + " Event: " + str(query_action))
# WMI event
if event["Event"]["System"]["EventID"] == 19 or event["Event"]["System"]["EventID"] == 20 \
or event["Event"]["System"]["EventID"] == 21:
logger.info("ToDo")
# DNS
if event["Event"]["System"]["EventID"] == 22:
try:
insert_process(cursor, event)
except Exception as e:
logger.error("Error insert_process 22: " + str(e) + " Event: " + str(event["Event"]))
try: # Query
query_dnsquery = 'INSERT INTO public."DNSQuery" ("QueryName") ' \
"VALUES ('{}') ON CONFLICT DO NOTHING;".format(
event["Event"]["EventData"]["QueryName"])
cursor.execute(query_dnsquery)
except Exception as e:
logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsquery))
try: # Resolution
query_dnsresolution = 'INSERT INTO public."DNSResolution" ("UtcTime","QueryName","QueryStatus","QueryResults") ' \
"VALUES ('{}','{}','{}','{}') ON CONFLICT DO NOTHING;".format(
event["Event"]["EventData"]["UtcTime"],
event["Event"]["EventData"]["QueryName"],
event["Event"]["EventData"]["QueryStatus"],
event["Event"]["EventData"]["QueryResults"])
cursor.execute(query_dnsresolution)
except Exception as e:
logger.error("Error query_file 22: " + str(e) + " Event: " + str(query_dnsresolution))
try: # Action
query_action = 'INSERT INTO public."Actions" ("UtcTime","ActionType","ProcessGuid","DestinationId")' \
" VALUES ('{}','{}','{}','{}') ;".format(
event["Event"]["EventData"]["UtcTime"],
"DnsRequest",
(event["Event"]["EventData"]["Image"]).lower(),
event["Event"]["EventData"]["QueryName"])
cursor.execute(query_action)
except Exception as e:
logger.error("Error query_action 22: " + str(e) + " Event: " + str(query_action))
cursor.close()
end = time.time()
logger.info("Time to process file %s seconds ---" % (end - start))
def load_data(self):
self.__parser = PyEvtxParser(self.__filename)
for record in self.__parser.records():
self.__records[record["event_record_id"]] = EventRecord(record)
self.__record_ids.append(record["event_record_id"])
self.__record_ids.sort()
def __init__(self, filepath: str) -> None:
self.path = Path(filepath)
self.parser = PyEvtxParser(self.path.open(mode='rb'))
class Evtx2es(object):
def __init__(self, filepath: str) -> None:
self.path = Path(filepath)
self.parser = PyEvtxParser(self.path.open(mode='rb'))
def gen_json(self, size: int) -> Generator:
buffer: List[dict] = []
for record in self.parser.records_json():
record['data'] = json.loads(record.get('data'))
eventid_field = record.get('data').get('Event').get('System').get(
'EventID')
if type(eventid_field) is dict:
record['data']['Event']['System'][
'EventID'] = eventid_field.get('#text')
try:
status = record.get('data').get('Event').get('EventData').get(
'Status')
record['data']['Event']['EventData']['Status'] = None
except Exception:
pass
# Convert data according to ECS (sort of)
# First copy system fields
record['winlog'] = {
'channel':
record['data']['Event']['System']['Channel'],
'computer_name':
record['data']['Event']['System']['Computer'],
'event_id':
record['data']['Event']['System']['EventID'],
'opcode':
record['data']['Event']['System'].get('Opcode'),
'provider_guid':
record['data']['Event']['System']['Provider']
['#attributes'].get('Guid'),
'provider_name':
record['data']['Event']['System']['Provider']['#attributes']
['Name'],
'record_id':
record['data']['Event']['System']['EventRecordID'],
'task':
record['data']['Event']['System']['Task'],
'version':
record['data']['Event']['System'].get('Version'),
}
try:
record['winlog']['process'] = {
'pid':
record['data']['Event']['System']['Execution']
['#attributes']['ProcessID'],
'thread_id':
record['data']['Event']['System']['Execution']
['#attributes']['ThreadID'],
}
except KeyError:
pass
record.update({
'log': {
'file': {
'name': str(self.path)
}
},
'event': {
'code':
record['winlog']['event_id'],
'created':
record['data']['Event']['System']['TimeCreated']
['#attributes']['SystemTime'],
}
})
record['@timestamp'] = record['event']['created']
# Move event attributes to ECS location
record['winlog']['event_data'] = record['data']['Event'].get(
'EventData', dict())
del record['data']
if record['winlog']['event_data'] is None or len(
record['winlog']
['event_data']) == 0: # remove event_data fields if empty
del record['winlog']['event_data']
else:
for k, v in record['winlog']['event_data'].items():
# Normalize some known problematic fields with values switching between integers and strings with hexadecimal notation to integers
if k in ('ProcessId') and type(v) == str:
if v.startswith("0x"):
record['winlog']['event_data'][k] = int(v, 16)
else:
try:
record['winlog']['event_data'][k] = int(v)
except ValueError:
record['winlog']['event_data'][k] = 0
# Maximum limit of numeric values in Elasticsearch
if type(v) is int:
if v < -2**63:
record['winlog']['event_data'][k] = -2**63
elif v > 2**63 - 1:
record['winlog']['event_data'][k] = 2**63 - 1
buffer.append(record)
if len(buffer) >= size:
yield buffer
buffer.clear()
else:
yield buffer
def ProcessPrivateRules(self, logspath):
if self.logger:
self.logger.info(
f"Starting processing private rules on the log/s in '{logspath}' ..."
)
for pubrule in self.PublicRulesContainsPrivateRules:
triggered = None
privRules = []
privRulesChannels = []
TriggeredEvents = {}
for privrulename in pubrule.include.get("rule"):
for rule in self.ruleSet:
if rule.name == privrulename:
privRules.append(rule)
privRulesChannels.append(rule.channel.lower())
for filePath in self.LogsToProcess:
parser = PyEvtxParser(filePath)
for record in parser.records_json():
try:
data = json.loads(record["data"])
event = Event(data)
if event.Channel.lower() in privRulesChannels:
for prirule in privRules:
if self.match(prirule, event):
if triggered == None:
triggered = True
triggered = triggered and True
if not TriggeredEvents.get(prirule):
TriggeredEvents[prirule] = [event]
else:
TriggeredEvents[prirule].append(event)
else:
break
except (OSError, KeyError) as e:
if self.logger:
self.logger.error(e, exc_info=True)
continue
except Exception as e:
if self.logger:
self.logger.error(e, exc_info=True)
if len(TriggeredEvents) != len(privRules):
return False
TriggeredEventsList = []
for key, val in TriggeredEvents.items():
TriggeredEventsList.append(val)
TriggeredEventsWithinTheSpecifiedTime = self.ProcessTimeBetweenLogs(
list(itertools.product(*TriggeredEventsList)),
int(pubrule.include.get("if").get("within")))
if TriggeredEventsWithinTheSpecifiedTime:
for EventSet in TriggeredEventsWithinTheSpecifiedTime:
recordIDs = []
triggeredEventsData = {}
privRuleNames = pubrule.include.get("rule")
privateRules = []
for r in self.ruleSet:
if r.name in privRuleNames:
privateRules.append(r)
for r in privateRules:
for e in EventSet:
if r.channel == e.Channel:
if r.returns:
fields = {}
for field in r.returns:
fields.update(
{field: e.EventData.get(field)})
triggeredEventsData[r.name] = fields
else:
triggeredEventsData[r.name] = e.RawRecord
for event in EventSet:
recordIDs.append(event.EventRecordID)
data = [
event.TimeCreatedSystemTime, recordIDs, pubrule.name,
pubrule.score, pubrule.description, pubrule.reference,
[], triggeredEventsData
]
self.Queue.put(
Alert(event,
pubrule, [],
privateRule=True,
record=data))
else:
return False
def test_it_returns_error_on_non_existing_path():
with pytest.raises(FileNotFoundError):
parser = PyEvtxParser("non_existing")
def parseevtx(infile, outdir, coldelimiter, jsonfile):
outfilenametmp = os.path.basename(infile) + ".csv"
if jsonfile == 1:
jsonoutfilenametmp = os.path.basename(infile) + ".json"
drive, outfiledir = os.path.splitdrive(os.path.dirname(infile))
#Make Dir
harddir = outdir + os.sep + outfiledir
outfilename = harddir + os.sep + outfilenametmp
os.makedirs(harddir, exist_ok=True)
print(f'CVS Out Info : {outfilename}')
if jsonfile == 1:
jsonoutfilename = harddir + os.sep + jsonoutfilenametmp
print(f'JSON Out Info: {jsonoutfilename}')
jsonoutfile = open(jsonoutfilename, 'w')
# #1 #2 #3 #4 #5 #6 #7 #8 #9 #10 #11
header = "'EventID'|'ComputerName'|'TimeCreated'|'Security SID'|'UserID'|'Address'|'Script/IP:Port'|'Payload/Path/Correlation Activity ID'|'Event'|'Channel (EVTX Log)'|'SRC File'\n"
#create file
outfile = open(outfilename, 'w')
outfile.write(
"Please note that depending on the CSV deliminater used, the fields may not line up.\n"
)
outfile.write(header)
parser = PyEvtxParser(infile)
print(f'Working on: {infile}')
for record in parser.records_json():
subrec = json.loads(record["data"])
lineout = ""
try:
eventid = subrec['Event']['System']['EventID'] #['#text']
except:
eventid = subrec['Event']['System']['EventID']['#text']
channel = subrec['Event']['System']['Channel']
computer = subrec['Event']['System']['Computer']
systemtime = subrec['Event']['System']['TimeCreated']['#attributes'][
'SystemTime']
#print(subrec['Event']['UserData']['EventXML'].keys())
if jsonfile == 1:
if channel == "Windows PowerShell" or channel == "Microsoft-Windows-WinRM/Operational" or channel == "Microsoft-Windows-PowerShell/Operational" or channel == "Security" or channel == "System":
jsonoutfile.write(printjson(subrec))
jsonoutfile.write("\n")
#print(f'EID: {eventid}')
#--------------------------------------------------------------------------------------------------------------------------------
#Windows PowerShell.evtx
if eventid == 400 or eventid == 403 or eventid == 600:
if channel == "Windows PowerShell":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['Data']['#text'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
#Microsoft-Windows-WinRM/Operational.evtx
if eventid == 6:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Connecting remotely to: " + subrec[
'Event']['EventData'][
'connection'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 81:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Operation Name: " + subrec['Event'][
'EventData']['operationName'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 82:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Operation: " + subrec['Event'][
'EventData'][
'operation'] + " -- Resource URI: " + subrec[
'Event']['EventData'][
'resourceURI'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 134:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Operation: " + subrec['Event'][
'EventData']['operationName'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
# Need an example log related to powershell; now it prints out all 142 events
if eventid == 142:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'OperatinName: " + subrec['Event'][
'EventData'][
'operationName'] + " -- Error Code :'" + str(
subrec['Event']['EventData']
['errorCode']) + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 169:
if channel == "Microsoft-Windows-WinRM/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "'" + subrec['Event']['EventData'][
'username'] + "'" + coldelimiter
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "'Authentication Mechanism: " + subrec[
'Event']['EventData'][
'authenticationMechanism'] + "'" + coldelimiter #7
try:
lineout = lineout + "'" + subrec['Event']['System'][
'Correlation']['#attributes'][
'ActivityID'] + "'" + coldelimiter #8
except:
lineout = lineout + "''" + coldelimiter
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
#Microsoft-Windows-PowerShell/Operational.evtx
if eventid == 4100:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['ContextInfo'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "'" + subrec['Event']['EventData'][
'Payload'] + "'" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 4103:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec) ScriptBlockText
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['ContextInfo'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "'" + subrec['Event']['EventData'][
'Path'] + "'" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 4104:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
tmpstr = ''
tmpstr = tmpstr.join(
subrec['Event']['EventData']['ScriptBlockText'])
lineout = lineout + "'" + tmpstr + "'" + coldelimiter #7
lineout = lineout + "'" + subrec['Event']['EventData'][
'Path'] + "'" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
if eventid == 40961:
if channel == "Microsoft-Windows-PowerShell/Operational":
try:
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
#Security
if eventid == 4688:
if channel == "Security":
try:
if re.search(
'powershell',
subrec['Event']['EventData']['NewProcessName'],
re.IGNORECASE):
#printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['EventData'][
'SubjectDomainName'] + "\\" + subrec['Event'][
'EventData']['SubjectUserName'] + " (" + subrec[
'Event']['EventData'][
'SubjectUserSid'] + ")'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
try:
tmpstr = subrec['Event']['EventData'][
'CommandLine']
try:
tmpppn = subrec['Event']['EventData'][
'ParentProcessName']
except:
tmpppn = ''
if tmpstr:
lineout = lineout + "'" + tmpppn + " -> " + subrec[
'Event']['EventData'][
'CommandLine'] + "'" + coldelimiter #7
else:
lineout = lineout + "'" + tmpppn + " -> " + subrec[
'Event']['EventData'][
'NewProcessName'] + "'" + coldelimiter #7
except:
lineout = lineout + "'" + tmpppn + " -> " + subrec[
'Event']['EventData'][
'NewProcessName'] + "'" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
#--------------------------------------------------------------------------------------------------------------------------------
if eventid == 7030 or eventid == 7040 or eventid == 7045:
if channel == "System":
try:
printjson(subrec)
address = ''
lineout = "'" + str(eventid) + "'" + coldelimiter #1
lineout = lineout + "'" + computer + "'" + coldelimiter #2
lineout = lineout + "'" + systemtime + "'" + coldelimiter #3
lineout = lineout + "'" + subrec['Event']['System'][
'Security']['#attributes'][
'UserID'] + "'" + coldelimiter #4
lineout = lineout + "" + coldelimiter #5
lineout = lineout + "" + coldelimiter #6
lineout = lineout + "" + coldelimiter #7
lineout = lineout + "" + coldelimiter #8
lineout = lineout + "'" + eventidtxt[str(
eventid)] + "'" + coldelimiter #9
lineout = lineout + "'" + channel + "'" + coldelimiter #10
lineout = lineout + "'" + infile + "'" #11
lineout = lineout + "\n"
#print(f'{lineout}')
outfile.write(lineout)
except:
parseproblem(subrec, outfile)
print("end")
outfile.close()
if jsonfile == 1:
jsonoutfile.close()
def test_it_fails_on_file_opened_as_text(small_sample):
with pytest.raises(OSError) as e:
with open(small_sample, "rt") as o:
parser = PyEvtxParser(o)
assert "decode byte" in e.value.args[1]