def test_Filehitcount1(self): with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() entry_fields = settings.EntriesFields(EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='test123.exe') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields(EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='test1234.exe') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields(EntryType=settings.__APPCOMPAT__, FilePath='C:\\test123.exe', FileName='nohit.exe') add_entry(DB, "TestHost01", entry_fields) # Get temp db name for the test temp_file = tempfile.NamedTemporaryFile(suffix='.db', prefix='testCase', dir=tempfile.gettempdir()) temp_file.close() with open(temp_file.name, 'w') as fh: fh.write('test123.exe') try: ret = main([self.testset1, "filehitcount", temp_file.name]) except Exception as e: print traceback.format_exc() self.fail(e.message + "\n" + traceback.format_exc()) # Remove temp file os.remove(temp_file.name) num_hits = len(ret) self.assertEquals(num_hits, 2, sys._getframe().f_code.co_name) self.assertEquals(ret[1][1][1][0], 'test123.exe', "test_Tstomp1 failed!") self.assertEquals(int(ret[1][1][1][1]), 1, "test_Tstomp1 failed!")
def test_AppCompat_LiteralSearch_Suppressed(self): rndFileName = ''.join( random.choice(string.ascii_uppercase) for _ in range(15)) with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() # Add 10 entries for i in xrange(0, 10): entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName=rndFileName, Size=i, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) # Add 10 entries which will be deduped to 1 on search for i in xrange(0, 10): entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName=rndFileName, Size=1000, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) # Get temp file name for the DB with tempfile.NamedTemporaryFile( suffix='.txt', prefix='Output', dir=tempfile.gettempdir()) as temp_file: # Search (num_hits, num_hits_suppressed, results) = main([ "-o", temp_file.name, self.testset1, "search", "-F", rndFileName ]) # Check we got as many hits as we expect self.assertTrue( num_hits == 10 + 10, sys._getframe().f_code.co_name + " num_hits: %d - %s" % (num_hits, self.testset1)) # Check supression worked as expected self.assertTrue( num_hits_suppressed == 9, sys._getframe().f_code.co_name + " num_hits: %d" % num_hits) # Check output has the expected result self.assertEquals( num_hits - num_hits_suppressed, self.count_lines_regex(temp_file.name, rndFileName), sys._getframe().f_code.co_name + " Output regex count doesn't match num_hits!")
def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 rowValid = True file_object = loadFile(file_fullpath) csvdata = file_object.read().splitlines()[1:] file_object.close() data = csv.reader(csvdata, dialect='IngestDialect1') for row in data: for field in row: if b'\x00' in field: settings.logger.warning( "NULL byte found, ignoring bad shimcache parse: %s" % field) rowValid = False if rowValid: path, filename = ntpath.split(row[2]) namedrow = settings.EntriesFields( HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, LastModified=unicode(row[0]), LastUpdate=unicode(row[1]), FilePath=unicode(path), FileName=unicode(filename), Size=unicode(row[3]), ExecFlag=str(row[4]), InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1
def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 file_object = loadFile(file_fullpath) rows = file_object.read().splitlines()[1:] file_object.close() appCompatREGEX = re.compile( "((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ]((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ](.*)\\\([^\\\]*)[, ](N\/A|\d*)[, ](N\/A|True|False)" ) assert (rows is not None) for r in rows: if b'\x00' in r: logger.debug( "NULL byte found, ignoring bad shimcache parse: %s" % r) continue m = appCompatREGEX.match(r) if m: namedrow = settings.EntriesFields( HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, LastModified=unicode(m.group(1)), LastUpdate=unicode(m.group(2)), FilePath=unicode(m.group(3)), FileName=unicode(m.group(4)), Size=unicode(m.group(5)), ExecFlag=str(m.group(6)), InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1 else: logger.warning("Entry regex failed for: %s - %s" % (hostID, r))
def test_AmCache_LiteralSearch(self): with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() for i in xrange(0, 10): entry_fields = settings.EntriesFields( EntryType=settings.__AMCACHE__, FilePath='C:\Temp', FileName='calc.exe', Size=i, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) # Get temp file name for the DB with tempfile.NamedTemporaryFile( suffix='.txt', prefix='Output', dir=tempfile.gettempdir()) as temp_file: # Search (num_hits, num_hits_suppressed, results) = main([ "-o", temp_file.name, self.testset1, "search", "-F", "calc.exe" ]) # Check we got at least as many as we added into the DB self.assertTrue( num_hits >= 10, sys._getframe().f_code.co_name + " num_hits: %d" % num_hits) # Check output has the expected result self.assertEquals( num_hits, self.count_lines_regex(temp_file.name, "calc\.exe"), sys._getframe().f_code.co_name + " Output regex count doesn't match num_hits!")
def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 file_object = loadFile(file_fullpath) rows = _processAmCacheFile_StringIO(file_object) file_object.close() for r in rows: namedrow = settings.EntriesFields( HostID=hostID, EntryType=settings.__AMCACHE__, RowNumber=rowNumber, FilePath=(None if r.path == None else ntpath.dirname(r.path)), FileName=(None if r.path == None else ntpath.basename(r.path)), Size=r.size, ExecFlag='True', SHA1=(None if r.sha1 == None else r.sha1[4:]), FileDescription=r.file_description, FirstRun=r.first_run, Created=r.created_timestamp, Modified1=r.modified_timestamp, Modified2=r.modified_timestamp2, LinkerTS=r.linker_timestamp, Product=r.product, Company=r.company, PE_sizeofimage=r.pe_sizeofimage, Version_number=r.version_number, Version=r.version, Language=r.language, Header_hash=r.header_hash, PE_checksum=r.pe_checksum, SwitchBackContext=r.switchbackcontext, InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1
def test_TStack(self): rndFileName = 'randomfilename.rnd' with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() # Add stuff to stack for i in xrange(0, 10): entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Windows', FileName=rndFileName, Size=i, LastModified='1000-01-01 00:00:0' + str(i)) add_entry(DB, "TestHost01", entry_fields) # Run ret = main([self.testset1, "tstack", '1000-01-01', '1000-01-02']) # Check we found the right file self.assertEquals(ret[1][1][0], rndFileName, "test_TStack failed!") # Check expected in count self.assertEquals(int(ret[1][1][1]), 10, "test_TStack failed!") # Check expected out count self.assertEquals(int(ret[1][1][2]), 0, "test_TStack failed!")
def test_Stack(self): rndFileName = ''.join( random.choice(string.ascii_uppercase) for _ in range(15)) with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() # Add stuff to stack for i in xrange(0, 10): entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Windows', FileName=rndFileName, Size=i, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) # Run ret = main([ self.testset1, "stack", "FileName", "FilePath = 'c:\Windows'" ]) # Check status count == db count count = int([i[1][0] for i in ret if rndFileName in i[1]][0]) self.assertEquals(count, 10, "test_Stack failed!")
def test_AppCompat_IndexedSearch2(self): rndFileName = ''.join( random.choice(string.ascii_uppercase) for _ in range(20)) with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() for i in xrange(0, 20): entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName=rndFileName, Size=i, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) # Get temp file name for the DB with tempfile.NamedTemporaryFile( suffix='.txt', prefix='test_AppCompat_IndexedSearch', dir=tempfile.gettempdir()) as temp_file_indexed: with tempfile.NamedTemporaryFile( suffix='.txt', prefix='test_AppCompat_NormalSearch', dir=tempfile.gettempdir()) as temp_file_normal: # Indexed Search (num_hits, num_hits_suppressed, results) = main([ "-o", temp_file_indexed.name, self.testset1, "fsearch", "FileName", "-F", rndFileName ]) # Standard Search (num_hits2, num_hits_suppressed2, results2) = main([ "-o", temp_file_normal.name, self.testset1, "search", "-F", "\\" + rndFileName ]) # Check we got the same number of hits self.assertTrue( num_hits == num_hits2, sys._getframe().f_code.co_name + " num_hits: %d" % num_hits) # Check output has the expected results self.assertEquals( num_hits - num_hits_suppressed, self.count_lines_regex(temp_file_indexed.name, rndFileName), sys._getframe().f_code.co_name + " Output regex count doesn't match num_hits!") # Check output has the expected results self.assertEquals( num_hits2 - num_hits_suppressed2, self.count_lines_regex(temp_file_normal.name, rndFileName), sys._getframe().f_code.co_name + " Output regex count doesn't match num_hits!") # Check standard and indexed search produced the same results self.assertTrue( self.compare_output_files(temp_file_normal.name, temp_file_indexed.name), "Results differ!")
def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 rowValid = True minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0) maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0) file_object = loadFile(file_fullpath) csvdata = file_object.read().splitlines()[1:] file_object.close() data = csv.reader(csvdata, dialect='IngestDialect1') for row in data: for field in row: if b'\x00' in field: settings.logger.warning( "NULL byte found, ignoring bad shimcache parse: %s" % field) rowValid = False try: # Convert to timestamps: if row[0] != 'N/A': tmp_LastModified = datetime.strptime( row[0], "%Y-%m-%d %H:%M:%S") else: tmp_LastModified = minSQLiteDTS if row[1] != 'N/A': tmp_LastUpdate = datetime.strptime( row[1], "%Y-%m-%d %H:%M:%S") else: tmp_LastUpdate = minSQLiteDTS except Exception as e: print("crap") exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.info( "Exception processing row (%s): %s [%s / %s / %s]" % (e.message, unicode(ntpath.split( row[2])[0]), exc_type, fname, exc_tb.tb_lineno)) if rowValid: path, filename = ntpath.split(row[2]) namedrow = settings.EntriesFields( HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, LastModified=tmp_LastModified, LastUpdate=tmp_LastUpdate, FilePath=unicode(path), FileName=unicode(filename), Size=unicode(row[3]), ExecFlag=str(row[4]), InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1
def test_Leven2(self): with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() # Add stuff entry_fields = settings.EntriesFields(EntryType=settings.__APPCOMPAT__, FilePath='C:\Windows\System32', FileName='svchosts.exe') add_entry(DB, "TestHost01", entry_fields) # Run ret = main([self.testset1, "leven"]) # Check we found the right file self.assertEquals('svchosts.exe' in ret[1][1][1], True, "test_Leven2 failed!")
def test_Leven(self): rndFileName = ''.join(random.choice(string.ascii_uppercase) for _ in range(15)) with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() # Add stuff entry_fields = settings.EntriesFields(EntryType=settings.__APPCOMPAT__, FilePath='C:\Windows\System32', FileName=rndFileName) add_entry(DB, "TestHost01", entry_fields) # Run leven_fileName = 'a' + rndFileName ret = main([self.testset1, "leven", leven_fileName]) # Check we found the right file self.assertEquals(ret[1][1][1], "'"+rndFileName+"'", "test_Leven failed!")
def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 check_tags = ['LastModified', 'AppCompatPath', 'ExecutionFlag'] try: xml_data = loadFile(file_fullpath) for event, element in etree.iterparse(xml_data, events=("end",)): skip_entry = False tag_dict = {} if element.tag == "AppCompatItemExtended": self._processElement(element, tag_dict) # Check we have everything we need and ignore entries with critical XML errors on them for tag in check_tags: if tag not in tag_dict or tag_dict[tag] is None: if 'AppCompatPath' in tag_dict: logger.warning("Malformed tag [%s: %s] in %s, entry: %s (skipping entry)" % (tag, tag_dict[tag], tag_dict['AppCompatPath'], file_fullpath)) else: logger.warning( "Malformed tag [%s: %s] in %s, entry: Unknown (skipping entry)" % (tag, tag_dict[tag], file_fullpath)) skip_entry = True break # If the entry is valid do some housekeeping: if not skip_entry: if tag_dict['ExecutionFlag'] == '1': tmpExexFlag = True elif tag_dict['ExecutionFlag'] == '0': tmpExexFlag = False else: tmpExexFlag = tag_dict['ExecutionFlag'] namedrow = settings.EntriesFields(HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, InstanceID=instanceID, LastModified=(tag_dict['LastModified'].replace("T"," ").replace("Z","") if 'LastModified' in tag_dict else '0001-01-01 00:00:00'), LastUpdate=(tag_dict['LastUpdate'].replace("T"," ").replace("Z","") if 'LastUpdate' in tag_dict else '0001-01-01 00:00:00'), FileName=ntpath.basename(tag_dict['AppCompatPath']), FilePath=ntpath.dirname(tag_dict['AppCompatPath']), Size=(tag_dict['Size'] if 'Size' in tag_dict else 'N/A'), ExecFlag=tmpExexFlag) rowsData.append(namedrow) rowNumber += 1 else: pass element.clear() xml_data.close() except Exception as e: print e.message print traceback.format_exc() pass
def test_AppCompat_IndexedSearchFilePath(self): rndFileName = ''.join( random.choice(string.ascii_uppercase) for _ in range(20)) with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() for i in xrange(0, 20): entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\\' + rndFileName, FileName="calc.exe", Size=i, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) # Get temp file name for the DB with tempfile.NamedTemporaryFile( suffix='.txt', prefix='test_AppCompat_IndexedSearch', dir=tempfile.gettempdir()) as temp_file: # Search (num_hits, num_hits_suppressed, results) = main([ "-o", temp_file.name, self.testset1, "fsearch", "FilePath", "-F", "C:\\" + rndFileName ]) # Check we got at least as many as we added into the DB self.assertTrue( num_hits == 20, sys._getframe().f_code.co_name + " num_hits: %d" % num_hits) # Check output has the expected result self.assertEquals( num_hits - num_hits_suppressed, self.count_lines_regex(temp_file.name, rndFileName), sys._getframe().f_code.co_name + " Output regex count doesn't match num_hits!")
def processFile(self, file_fullpath, hostID, instanceID, rowsData): minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0) maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0) rowNumber = 0 check_tags = ['LastModified', 'AppCompatPath'] try: xml_data = loadFile(file_fullpath) for event, element in etree.iterparse(xml_data, events=("end", )): skip_entry = False tag_dict = {} if element.tag == "AppCompatItemExtended": self._processElement(element, tag_dict) # From time to time we get some entries with no real data on them for some unknown reason, skip for now if 'AppCompatPath' in tag_dict: if tag_dict['AppCompatPath'] == 'N/A': logger.debug( "ShimCache entry with no AppCompatPath (Sequence # %s) on %s. (skipping entry)" % (tag_dict['Sequence'], file_fullpath)) break # Check we have everything we need and ignore entries with critical XML errors on them for tag in check_tags: if tag not in tag_dict or tag_dict[tag] is None: if tag not in tag_dict: if 'AppCompatPath' in tag_dict: logger.warning( "Missing tag [%s] in %s, entry: %s (skipping entry)" % (tag, tag_dict['AppCompatPath'], file_fullpath)) else: logger.warning( "Malformed tag [%s] in %s, entry: Unknown (skipping entry)" % (tag, file_fullpath)) skip_entry = True break if tag_dict[tag] is None: if 'AppCompatPath' in tag_dict: logger.warning( "Malformed tag [%s: %s] in %s, entry: %s (skipping entry)" % (tag, tag_dict[tag], tag_dict['AppCompatPath'], file_fullpath)) else: logger.warning( "Malformed tag [%s: %s] in %s, entry: Unknown (skipping entry)" % (tag, tag_dict[tag], file_fullpath)) skip_entry = True break # If the entry is valid do some housekeeping: if not skip_entry: if tag_dict['ExecutionFlag'] == '1': tmpExecFlag = True elif tag_dict['ExecutionFlag'] == '0': tmpExecFlag = False else: tmpExecFlag = tag_dict['ExecutionFlag'] try: # Convert TS to datetime format if 'LastModified' in tag_dict: tmp_LastModified = tag_dict[ 'LastModified'].replace("T", " ").replace( "Z", "") if type(tmp_LastModified) is not datetime: tmp_LastModified = datetime.strptime( tmp_LastModified, "%Y-%m-%d %H:%M:%S") else: tmp_LastModified = minSQLiteDTS if 'LastUpdate' in tag_dict: tmp_LastUpdate = tag_dict[ 'LastUpdate'].replace("T", " ").replace( "Z", "") if type(tmp_LastUpdate) is not datetime: tmp_LastUpdate = datetime.strptime( tmp_LastUpdate, "%Y-%m-%d %H:%M:%S") else: tmp_LastUpdate = minSQLiteDTS namedrow = settings.EntriesFields( HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, InstanceID=instanceID, LastModified=tmp_LastModified, LastUpdate=tmp_LastUpdate, FileName=ntpath.basename( tag_dict['AppCompatPath']), FilePath=ntpath.dirname( tag_dict['AppCompatPath']), Size=(tag_dict['Size'] if 'Size' in tag_dict else 'N/A'), ExecFlag=tmpExecFlag) rowsData.append(namedrow) rowNumber += 1 except Exception as e: print("crap") exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.info( "Exception processing row (%s): %s [%s / %s / %s]" % (e.message, file_fullpath, exc_type, fname, exc_tb.tb_lineno)) else: pass element.clear() xml_data.close() except Exception as e: print e.message print traceback.format_exc() pass
class Appcompat_miracquisition(Ingest): ingest_type = "appcompat_miracquisition" file_name_filter = "(?:.*)(?:\/|\\\)(.*)-[A-Za-z0-9]{64}-\d{1,10}-\d{1,10}(?:_octet-stream.xml)$" def __init__(self): super(Appcompat_miracquisition, self).__init__() def getHostName(self, file_name_fullpath): host_name_from_file = super(Appcompat_miracquisition, self).getHostName(file_name_fullpath) file_object = loadFile(file_name_fullpath) regf_file = pyregf.file() regf_file.open_file_object(file_object, "r") # Get control set number tmp_key = regf_file.get_key_by_path(r'Select') if tmp_key is not None: controlset_number = tmp_key.get_value_by_name( 'Current').get_data_as_integer() # Get host name tmp_key = regf_file.get_key_by_path( r'ControlSet00' + str(controlset_number) + '\Control\ComputerName\ComputerName') host_name = tmp_key.get_value_by_name( 'ComputerName').get_data_as_string() # Check Mir host name matches the one from the hive, trust but verify :) if host_name != host_name_from_file: logger.warning( "Host name mismatch! (%s != %s): %s" % (host_name, host_name_from_file, file_name_fullpath)) else: # todo: Close everything down elegantly logger.error( "Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin: %s" % file_name_fullpath) raise (Exception( 'Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin' )) # Need to close these or the memory will never get freed: regf_file.close() del regf_file file_object.close() del file_object return host_name def checkMagic(self, file_name_fullpath): magic_ok = False # Quick and dirty check file_object = loadFile(file_name_fullpath) tmp = struct.unpack('4s', file_object.read(4)) if tmp[0] == "regf": # Perform a deeper check using pyregf regf_file = pyregf.file() regf_file.open_file_object(file_object, "r") magic_key = regf_file.get_key_by_path(r'Select') regf_file.close() del regf_file if magic_key is not None: magic_ok = True # Need to close these or the memory will never get freed: file_object.close() del file_object return magic_ok def calculateID(self, file_name_fullpath): instanceID = 0 file_object = loadFile(file_name_fullpath) regf_file = pyregf.file() regf_file.open_file_object(file_object, "r") # Search for key containing ShimCache entries on all control sets # Use last modification time of the last modified one as instanceID root = regf_file.get_root_key() num_keys = root.get_number_of_sub_keys() for i in xrange(0, num_keys): tmp_key = root.get_sub_key(i) if "controlset" in tmp_key.get_name().lower(): session_man_key = regf_file.get_key_by_path( "%s\Control\Session Manager" % tmp_key.get_name()) num_keys = session_man_key.get_number_of_sub_keys() for i in xrange(0, num_keys): tmp_key = session_man_key.get_sub_key(i) if "appcompatibility" in tmp_key.get_name().lower( ) or "appcompatcache" in tmp_key.get_name().lower(): last_write_time = tmp_key.get_last_written_time_as_integer( ) if last_write_time > instanceID: instanceID = last_write_time break # Need to close these or the memory will never get freed: regf_file.close() del regf_file file_object.close() del file_object return instanceID def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 entries = None # Process file using ShimCacheParser try: entries = read_from_hive(file_fullpath, True) if not entries: logger.warning("[ShimCacheParser] found no entries for %s" % file_fullpath) return False else: rows = write_it(entries, "StringIO")[1:] except IOError, err: logger.error("[ShimCacheParser] Error opening binary file: %s" % str(err)) # Process records appCompatREGEX = re.compile( "((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ]((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ](.*)\\\([^\\\]*)[, ](N\/A|\d*)[, ](N\/A|True|False)" ) assert (rows is not None) for r in rows: if b'\x00' in r: logger.debug( "NULL byte found, skipping bad shimcache parse: %s" % r) continue m = appCompatREGEX.match(r) if m: namedrow = settings.EntriesFields( HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, LastModified=unicode(m.group(1)), LastUpdate=unicode(m.group(2)), FilePath=unicode(m.group(3)), FileName=unicode(m.group(4)), Size=unicode(m.group(5)), ExecFlag=str(m.group(6)), InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1 else: logger.warning("Entry regex failed for: %s - %s" % (hostID, r))
class Appcompat_mirregistryaudit(Ingest): ingest_type = "appcompat_mirregistryaudit" file_name_filter = "(?:.*)(?:\/|\\\)(.*)-[A-Za-z0-9]{64}-\d{1,10}-\d{1,10}(?:_w32registry.xml)$" def __init__(self): super(Appcompat_mirregistryaudit, self).__init__() def calculateID(self, file_name_fullpath): instanceID = datetime.min tmp_instanceID = None try: file_object = loadFile(file_name_fullpath) root = ET.parse(file_object).getroot() file_object.close() for reg_key in root.findall('RegistryItem'): tmp_reg_key = reg_key.find('Modified') if tmp_reg_key is not None: reg_modified = tmp_reg_key.text try: tmp_instanceID = datetime.strptime(reg_modified, "%Y-%m-%dT%H:%M:%SZ") except ValueError as e: tmp_instanceID = datetime.max logger.warning("Invalid reg_modified date found!: %s (%s)" % (reg_modified, file_name_fullpath)) if instanceID < tmp_instanceID: instanceID = tmp_instanceID else: logger.warning("Found RegistryItem with no Modified date (Mir bug?): %s" % file_name_fullpath) except Exception: logger.exception("Error on calculateID for: %s" % file_name_fullpath) # If we found no Modified date in any of the RegistryItems we go with plan B (but most probably ShimCacheParser will fail to parse anyway) if instanceID is None: file_object = loadFile(file_name_fullpath) content = file_object.read() instanceID = hashlib.md5(content).hexdigest() file_object.close() return instanceID def checkMagic(self, file_name_fullpath): # As long as we find one AppcompatCache key we're declaring it good for us file_object = loadFile(file_name_fullpath) try: root = ET.parse(file_object).getroot() for reg_key in root.findall('RegistryItem'): if reg_key.find('ValueName').text == "AppCompatCache": return True except Exception: logger.warning("[%s] Failed to parse XML for: %s" % (self.ingest_type, file_name_fullpath)) #traceback.print_exc(file=sys.stdout) finally: file_object.close() return False def processFile(self, file_fullpath, hostID, instanceID, rowsData): # Returns data in rowsData rowNumber = 0 # Process file using ShimCacheParser try: xml_data = loadFile(file_fullpath) (error, entries) = read_mir(xml_data, True) xml_data.close() assert(not error) if not entries: logger.warning("[ShimCacheParser] found no entries for %s" % file_fullpath) return False else: rows = write_it(entries, "StringIO")[1:] except IOError, err: logger.error("[ShimCacheParser] Error opening binary file: %s" % str(err)) # Process records appCompatREGEX = re.compile( "((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ]((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ](.*)\\\([^\\\]*)[, ](N\/A|\d*)[, ](N\/A|True|False)") assert (rows is not None) for r in rows: if b'\x00' in r: logger.debug("NULL byte found, skipping bad shimcache parse: %s" % r) continue m = appCompatREGEX.match(r) if m: namedrow = settings.EntriesFields(HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, LastModified=unicode(m.group(1)), LastUpdate=unicode(m.group(2)), FilePath=unicode(m.group(3)), FileName=unicode(m.group(4)), Size=unicode(m.group(5)), ExecFlag=str(m.group(6)), InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1 else: logger.warning("Entry regex failed for: %s - %s" % (hostID, r))
def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 check_tags = ['AmCacheLastModified2'] try: xml_data = loadFile(file_fullpath) for event, element in etree.iterparse(xml_data, events=("end",)): skip_entry = False tag_dict = {} if element.tag == "AmCacheItem": self._processElement(element, tag_dict) # Check we have everything we need and ignore entries with critical XML errors on them for tag in check_tags: if tag not in tag_dict: if 'AmCacheFilePath' in tag_dict: logger.warning("Missing tag [%s] in %s, entry: %s (skipping entry)" % (tag, tag_dict['AmCacheFilePath'], file_fullpath)) else: logger.warning("Malformed tag [%s] in %s, entry: Unknown (skipping entry)" % (tag, file_fullpath)) skip_entry = True break if tag_dict[tag] is None: if 'AmCacheFilePath' in tag_dict: logger.warning("Malformed tag [%s: %s] in %s, entry: %s (skipping entry)" % (tag, tag_dict[tag], tag_dict['AmCacheFilePath'], file_fullpath)) else: logger.warning("Malformed tag [%s: %s] in %s, entry: Unknown (skipping entry)" % (tag, tag_dict[tag], file_fullpath)) skip_entry = True break # Some entries in AmCache do not refer to files per se (like installed program entries) # We don't have much use for them right now but let's keep the data there until I figure what to do with them if 'AmCacheFilePath' not in tag_dict: if 'ProgramName' in tag_dict: tag_dict['AmCacheFilePath'] = tag_dict['ProgramName'] else: # If we have no thing we can use here we skip the entry for now # todo: pretty-print the tag_dict to the log file logger.warning("AmCache entry with no AppCompatPath or ProgramName. (skipping entry)") break # If the entry is valid do some housekeeping: if not skip_entry: if 'ExecutionFlag' in tag_dict: if tag_dict['ExecutionFlag'] == '1': tmpExecFlag = True elif tag_dict['ExecutionFlag'] == '0': tmpExecFlag = False else: tmpExecFlag = tag_dict['ExecutionFlag'] else: # todo: Not all OS's have exec flag. Need to change the schema to reflect those cases! tmpExecFlag = False namedrow = settings.EntriesFields(HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, InstanceID=instanceID, LastModified=(tag_dict['LastModified'].replace("T"," ").replace("Z","") if 'LastModified' in tag_dict else '0001-01-01 00:00:00'), LastUpdate=(tag_dict['LastUpdate'].replace("T"," ").replace("Z","") if 'LastUpdate' in tag_dict else '0001-01-01 00:00:00'), FileName=ntpath.basename(tag_dict['AmCacheFilePath']), FilePath=ntpath.dirname(tag_dict['AmCacheFilePath']), Size=(tag_dict['Size'] if 'Size' in tag_dict else 'N/A'), ExecFlag=tmpExecFlag) rowsData.append(namedrow) rowNumber += 1 else: pass element.clear() xml_data.close() except Exception as e: print e.message print traceback.format_exc() pass
def test_TcorrTest_prog1(self): with appDB.DBClass(self.testset1, settings.__version__) as DB: DB.appInitDB() conn = DB.appConnectDB() # TestHost01 entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='AAA.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='BBB.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='CCC.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='DDD.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='EEE.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='FFF.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='GGG.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost01", entry_fields) # TestHost02 entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='AAA.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost02", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='BBB.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost02", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='CCC.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost02", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='DDD.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost02", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='EEE.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost02", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='FFF.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost02", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='GGG.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost02", entry_fields) try: directCorrelationData = main( [self.testset1, "tcorr", "DDD.exe", "-w 1"]) except Exception as e: print traceback.format_exc() self.fail(e.message + "\n" + traceback.format_exc()) # Check Names self.assertEquals(directCorrelationData[1][3], "CCC.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[0][3], "EEE.exe", "test_TcorrTest_prog1 - Name failed!") # Check Before self.assertEquals(directCorrelationData[1][6], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[0][6], 2, "test_TcorrTest_prog1 - Name failed!") # Check After self.assertEquals(directCorrelationData[1][7], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[0][7], 0, "test_TcorrTest_prog1 - Name failed!") # Check InvBond self.assertEquals(directCorrelationData[1][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[0][9], "True", "test_TcorrTest_prog1 - Name failed!") # Check Total_Count self.assertEquals(directCorrelationData[1][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[0][10], 2, "test_TcorrTest_prog1 - Name failed!") try: directCorrelationData = main( [self.testset1, "tcorr", "DDD.exe", "-w 2"]) except Exception as e: print traceback.format_exc() self.fail(e.message + "\n" + traceback.format_exc()) # Check Names self.assertEquals(directCorrelationData[0][3], "CCC.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][3], "EEE.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][3], "BBB.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][3], "FFF.exe", "test_TcorrTest_prog1 - Name failed!") # Check Before self.assertEquals(directCorrelationData[0][6], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][6], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][6], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][6], 2, "test_TcorrTest_prog1 - Name failed!") # Check After self.assertEquals(directCorrelationData[0][7], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][7], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][7], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][7], 0, "test_TcorrTest_prog1 - Name failed!") # Check InvBond self.assertEquals(directCorrelationData[0][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][9], "True", "test_TcorrTest_prog1 - Name failed!") # Check Total_Count self.assertEquals(directCorrelationData[0][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][10], 2, "test_TcorrTest_prog1 - Name failed!") # Check Weight self.assertTrue( directCorrelationData[0][8] > directCorrelationData[2][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[0][8] > directCorrelationData[3][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[1][8] > directCorrelationData[2][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[1][8] > directCorrelationData[3][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[0][8] == directCorrelationData[1][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[2][8] == directCorrelationData[3][8], "test_TcorrTest_prog1 - Name failed!") try: directCorrelationData = main( [self.testset1, "tcorr", "DDD.exe", "-w 3"]) except Exception as e: print traceback.format_exc() self.fail(e.message + "\n" + traceback.format_exc()) # Check Names self.assertEquals(directCorrelationData[0][3], "CCC.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][3], "EEE.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][3], "BBB.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][3], "FFF.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[4][3], "AAA.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[5][3], "GGG.exe", "test_TcorrTest_prog1 - Name failed!") # Check Before self.assertEquals(directCorrelationData[0][6], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][6], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][6], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][6], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[4][6], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[5][6], 2, "test_TcorrTest_prog1 - Name failed!") # Check After self.assertEquals(directCorrelationData[0][7], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][7], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][7], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][7], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[4][7], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[5][7], 0, "test_TcorrTest_prog1 - Name failed!") # Check InvBond self.assertEquals(directCorrelationData[0][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[4][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[5][9], "True", "test_TcorrTest_prog1 - Name failed!") # Check Total_Count self.assertEquals(directCorrelationData[0][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[2][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[3][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[4][10], 2, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[5][10], 2, "test_TcorrTest_prog1 - Name failed!") # Check Weight self.assertTrue( directCorrelationData[0][8] > directCorrelationData[2][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[0][8] > directCorrelationData[3][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[0][8] > directCorrelationData[4][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[0][8] > directCorrelationData[5][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[1][8] > directCorrelationData[2][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[1][8] > directCorrelationData[3][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[1][8] > directCorrelationData[4][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[1][8] > directCorrelationData[5][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[0][8] == directCorrelationData[1][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[2][8] == directCorrelationData[3][8], "test_TcorrTest_prog1 - Name failed!") self.assertTrue( directCorrelationData[4][8] == directCorrelationData[5][8], "test_TcorrTest_prog1 - Name failed!") # TestHost03 entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='AAA.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost03", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='BBB.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost03", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='CCC.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost03", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='DDD.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost03", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='EEE.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost03", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='FFF.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost03", entry_fields) entry_fields = settings.EntriesFields( EntryType=settings.__APPCOMPAT__, FilePath='C:\Temp', FileName='GGG.exe', Size=1, ExecFlag='True') add_entry(DB, "TestHost03", entry_fields) try: directCorrelationData = main( [self.testset1, "tcorr", "DDD.exe", "-w 1"]) except Exception as e: print traceback.format_exc() self.fail(e.message + "\n" + traceback.format_exc()) # Check Names self.assertEquals(directCorrelationData[0][3], "CCC.exe", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][3], "EEE.exe", "test_TcorrTest_prog1 - Name failed!") # Check Before self.assertEquals(directCorrelationData[0][6], 0, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][6], 3, "test_TcorrTest_prog1 - Name failed!") # Check After self.assertEquals(directCorrelationData[0][7], 3, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][7], 0, "test_TcorrTest_prog1 - Name failed!") # Check InvBond self.assertEquals(directCorrelationData[0][9], "True", "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][9], "True", "test_TcorrTest_prog1 - Name failed!") # Check Total_Count self.assertEquals(directCorrelationData[0][10], 3, "test_TcorrTest_prog1 - Name failed!") self.assertEquals(directCorrelationData[1][10], 3, "test_TcorrTest_prog1 - Name failed!")
class Appcompat_Raw_hive(Ingest): ingest_type = "appcompat_raw_hive" file_name_filter = "(?:.*)(?:\/|\\\)SYSTEM.*$" def __init__(self): super(Appcompat_Raw_hive, self).__init__() def getHostName(self, file_name_fullpath): file_object = loadFile(file_name_fullpath) regf_file = pyregf.file() regf_file.open_file_object(file_object, "r") # Get control set number tmp_key = regf_file.get_key_by_path(r'Select') if tmp_key is not None: controlset_number = tmp_key.get_value_by_name( 'Current').get_data_as_integer() # Get host name tmp_key = regf_file.get_key_by_path( r'ControlSet00' + str(controlset_number) + '\Control\ComputerName\ComputerName') host_name = tmp_key.get_value_by_name( 'ComputerName').get_data_as_string() else: # todo: Close everything down elegantly logger.error( "Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin: %s" % file_name_fullpath) raise (Exception( 'Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin' )) # Need to close these or the memory will never get freed: regf_file.close() del regf_file file_object.close() del file_object return host_name def checkMagic(self, file_name_fullpath): magic_ok = False # Check magic magic_id = self.id_filename(file_name_fullpath) if 'registry' in magic_id: file_object = loadFile(file_name_fullpath) regf_file = pyregf.file() regf_file.open_file_object(file_object, "r") magic_key = regf_file.get_key_by_path(r'Select') regf_file.close() del regf_file if magic_key is not None: magic_ok = True # Need to close these or the memory will never get freed: file_object.close() del file_object return magic_ok def calculateID(self, file_name_fullpath): instanceID = 0 file_object = loadFile(file_name_fullpath) regf_file = pyregf.file() regf_file.open_file_object(file_object, "r") # Search for key containing ShimCache entries on all control sets # Use last modification time of the last modified one as instanceID root = regf_file.get_root_key() num_keys = root.get_number_of_sub_keys() for i in xrange(0, num_keys): tmp_key = root.get_sub_key(i) if "controlset" in tmp_key.get_name().lower(): session_man_key = regf_file.get_key_by_path( "%s\Control\Session Manager" % tmp_key.get_name()) num_keys = session_man_key.get_number_of_sub_keys() for i in xrange(0, num_keys): tmp_key = session_man_key.get_sub_key(i) if "appcompatibility" in tmp_key.get_name().lower( ) or "appcompatcache" in tmp_key.get_name().lower(): last_write_time = tmp_key.get_last_written_time_as_integer( ) if last_write_time > instanceID: instanceID = last_write_time break # Need to close these or the memory will never get freed: regf_file.close() del regf_file file_object.close() del file_object return instanceID def processFile(self, file_fullpath, hostID, instanceID, rowsData): rowNumber = 0 entries = None minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0) maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0) # Process file using ShimCacheParser try: entries = read_from_hive(loadFile(file_fullpath), True) if not entries: logger.warning("[ShimCacheParser] found no entries for %s" % file_fullpath) return False else: rows = write_it(entries, "StringIO")[1:] except IOError, err: logger.error("[ShimCacheParser] Error opening binary file: %s" % str(err)) # Process records appCompatREGEX = re.compile( "((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ]((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ](.*)\\\([^\\\]*)[, ](N\/A|\d*)[, ](N\/A|True|False)" ) assert (rows is not None) for r in rows: if b'\x00' in r: logger.debug( "NULL byte found, skipping bad shimcache parse: %s" % r) continue m = appCompatREGEX.match(r) if m: try: # Convert to timestamps: if m.group(1) != 'N/A': tmp_LastModified = datetime.strptime( m.group(1), "%Y-%m-%d %H:%M:%S") else: tmp_LastModified = minSQLiteDTS if m.group(2) != 'N/A': tmp_LastUpdate = datetime.strptime( m.group(2), "%Y-%m-%d %H:%M:%S") else: tmp_LastUpdate = minSQLiteDTS except Exception as e: print("crap") exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.info( "Exception processing row (%s): %s [%s / %s / %s]" % (e.message, file_fullpath, exc_type, fname, exc_tb.tb_lineno)) namedrow = settings.EntriesFields( HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, LastModified=tmp_LastModified, LastUpdate=tmp_LastUpdate, FilePath=unicode(m.group(3)), FileName=unicode(m.group(4)), Size=unicode(m.group(5)), ExecFlag=str(m.group(6)), InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1 else: logger.warning("Entry regex failed for: %s - %s" % (hostID, r))
def processFile(self, file_fullpath, hostID, instanceID, rowsData): # Returns data in rowsData minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0) maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0) rowNumber = 0 check_tags = ['LastModified', 'AppCompatPath'] try: # Process file using ShimCacheParser try: xml_data = loadFile(file_fullpath) (error, entries) = read_mir(xml_data, True) xml_data.close() assert(not error) if not entries: logger.warning("[ShimCacheParser] found no entries for %s" % file_fullpath) return False else: rows = write_it(entries, "StringIO")[1:] except IOError, err: logger.error("[ShimCacheParser] Error opening binary file: %s" % str(err)) # Process records appCompatREGEX = re.compile( "((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ]((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ](.*)\\\([^\\\]*)[, ](N\/A|\d*)[, ](N\/A|True|False)") assert (rows is not None) for r in rows: if b'\x00' in r: logger.debug("NULL byte found, skipping bad shimcache parse: %s" % r) continue m = appCompatREGEX.match(r) if m: try: # Convert to timestamps: if m.group(1) != 'N/A': tmp_LastModified = datetime.strptime(m.group(1), "%Y-%m-%d %H:%M:%S") else: tmp_LastModified = minSQLiteDTS if m.group(2) != 'N/A': tmp_LastUpdate = datetime.strptime(m.group(2), "%Y-%m-%d %H:%M:%S") else: tmp_LastUpdate = minSQLiteDTS except Exception as e: print("crap") exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logger.info("Exception processing row (%s): %s [%s / %s / %s]" % ( e.message, file_fullpath, exc_type, fname, exc_tb.tb_lineno)) namedrow = settings.EntriesFields(HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber, LastModified=tmp_LastModified, LastUpdate=tmp_LastUpdate, FilePath=unicode(m.group(3)), FileName=unicode(m.group(4)), Size=unicode(m.group(5)), ExecFlag=str(m.group(6)), InstanceID=instanceID) rowsData.append(namedrow) rowNumber += 1 else: logger.warning("Entry regex failed for: %s - %s" % (hostID, r))
def processFile(self, file_fullpath, hostID, instanceID, rowsData): minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0) maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0) # alltags = set() tag_mapping = [ ('AmCacheCompanyName', 'N/A', 'string'), ('AmCacheCompileTime', 'N/A', 'datetime'), ('AmCacheCreated', 'N/A', 'datetime'), ('AmCacheFileDescription', 'N/A', 'string'), ('AmCacheFilePath', 'N/A', 'string'), ('AmCacheFileSize', 'N/A', 'string'), ('AmCacheFileVersionNumber', 'N/A', 'string'), ('AmCacheFileVersionString', 'N/A', 'string'), ('AmCacheLanguageCode', 'N/A', 'string'), ('AmCacheLastModified', 'N/A', 'datetime'), ('AmCacheRegLastModified', 'LastModified', 'datetime'), ('AmCachePEHeaderChecksum', 'N/A', 'string'), ('AmCachePEHeaderHash', 'N/A', 'string'), ('AmCachePEHeaderSize', 'N/A', 'string'), ('AmCacheProductName', 'N/A', 'string'), ('AmCacheProgramID', 'N/A', 'string'), ('AmCacheSha1', 'SHA1', 'string'), ('FileItem_Accessed', 'N/A', 'datetime'), ('FileItem_Changed', 'N/A', 'datetime'), ('FileItem_Created', 'N/A', 'datetime'), ('FileItem_DevicePath', 'N/A', 'string'), ('FileItem_Drive', 'N/A', 'string'), ('FileItem_FileAttributes', 'N/A', 'string'), ('FileItem_FileExtension', 'N/A', 'string'), ('FileItem_FileName', 'N/A', 'string'), ('FileItem_FilePath', 'N/A', 'string'), ('FileItem_FullPath', 'N/A', 'string'), ('FileItem_Md5sum', 'N/A', 'string'), ('FileItem_Modified', 'N/A', 'datetime'), ('FileItem_PEInfo_BaseAddress', 'N/A', 'string'), ('FileItem_PEInfo_DigitalSignature_Description', 'N/A', 'string'), ('FileItem_PEInfo_ExtraneousBytes', 'N/A', 'string'), ('FileItem_PEInfo_PEChecksum_PEComputedAPI', 'N/A', 'string'), ('FileItem_PEInfo_PEChecksum_PEFileAPI', 'N/A', 'string'), ('FileItem_PEInfo_PEChecksum_PEFileRaw', 'N/A', 'string'), ('FileItem_PEInfo_PETimeStamp', 'N/A', 'datetime'), ('FileItem_PEInfo_Subsystem', 'N/A', 'string'), ('FileItem_PEInfo_Type', 'N/A', 'string'), ('FileItem_SecurityID', 'N/A', 'string'), ('FileItem_SecurityType', 'N/A', 'string'), ('FileItem_SizeInBytes', 'N/A', 'string'), ('FileItem_Username', 'N/A', 'string'), ('ProgramEntryPresent', 'N/A', 'string'), ('ProgramInstallDate', 'N/A', 'datetime'), ('ProgramInstallSource', 'N/A', 'string'), ('ProgramLocaleID', 'N/A', 'string'), ('ProgramName', 'N/A', 'string'), ('ProgramPackageCode', 'N/A', 'string'), ('ProgramPackageCode2', 'N/A', 'string'), ('ProgramUninstallKey', 'N/A', 'string'), ('ProgramUnknownTimestamp', 'N/A', 'datetime'), ('ProgramVendorName', 'N/A', 'string'), ('ProgramVersion', 'N/A', 'string') ] rowNumber = 0 check_tags = ['AmCacheRegLastModified'] try: xml_data = loadFile(file_fullpath) for event, element in etree.iterparse(xml_data, events=("end", )): skip_entry = False tag_dict = {} if element.tag == "AmCacheItem": self._processElement(element, tag_dict) # Check we have everything we need and ignore entries with critical XML errors on them for tag in check_tags: if tag not in tag_dict: if 'AmCacheFilePath' in tag_dict: logger.warning( "Missing tag [%s] in %s, entry: %s (skipping entry)" % (tag, tag_dict['AmCacheFilePath'], file_fullpath)) else: logger.warning( "Malformed tag [%s] in %s, entry: Unknown (skipping entry)" % (tag, file_fullpath)) skip_entry = True break if tag_dict[tag] is None: if 'AmCacheFilePath' in tag_dict: logger.warning( "Malformed tag [%s: %s] in %s, entry: %s (skipping entry)" % (tag, tag_dict[tag], tag_dict['AmCacheFilePath'], file_fullpath)) else: logger.warning( "Malformed tag [%s: %s] in %s, entry: Unknown (skipping entry)" % (tag, tag_dict[tag], file_fullpath)) skip_entry = True break # Some entries in AmCache do not refer to files per se (like installed program entries) # We don't have much use for them right now but let's keep the data there until I figure what to do with them if 'AmCacheFilePath' not in tag_dict: if 'ProgramName' in tag_dict: tag_dict['AmCacheFilePath'] = tag_dict[ 'ProgramName'] else: # If we have nothing we can use here we skip the entry for now # todo: pretty-print the tag_dict to the log file logger.warning( "AmCache entry with no AppCompatPath or ProgramName. (skipping entry) %s" % file_fullpath) break # If the entry is valid do some housekeeping: if not skip_entry: if 'ExecutionFlag' in tag_dict: if tag_dict['ExecutionFlag'] == '1': tmpExecFlag = True elif tag_dict['ExecutionFlag'] == '0': tmpExecFlag = False else: tmpExecFlag = tag_dict['ExecutionFlag'] else: # todo: Not all OS's have exec flag. Need to change the schema to reflect those cases! tmpExecFlag = False try: # # Convert TS to datetime format # if 'LastModified' in tag_dict: # tmp_LastModified = tag_dict['LastModified'].replace("T", " ").replace("Z", "") # if type(tmp_LastModified) is not datetime: # tmp_LastModified = datetime.strptime(tmp_LastModified, "%Y-%m-%d %H:%M:%S") # else: tmp_LastModified = minSQLiteDTS # # if 'LastUpdate' in tag_dict: # tmp_LastUpdate = tag_dict['LastUpdate'].replace("T", " ").replace("Z", "") # if type(tmp_LastUpdate) is not datetime: # tmp_LastUpdate = datetime.strptime(tmp_LastUpdate, "%Y-%m-%d %H:%M:%S") # else: tmp_LastUpdate = minSQLiteDTS row_dict = {} row_dict['HostID'] = hostID row_dict['EntryType'] = settings.__APPCOMPAT__ row_dict['RowNumber'] = rowNumber row_dict['InstanceID'] = instanceID # row_dict['LastModified'] = tmp_LastModified # row_dict['LastUpdate'] = tmp_LastUpdate row_dict['FileName'] = ntpath.basename( tag_dict['AmCacheFilePath']) row_dict['FilePath'] = ntpath.dirname( tag_dict['AmCacheFilePath']) row_dict['Size'] = (tag_dict['Size'] if 'Size' in tag_dict else 'N/A') row_dict['ExecFlag'] = tmpExecFlag # for tag in tag_dict.keys(): # alltags.add(tag) # Add all tags available with mappings to our database schema for src_tag, dest_tag, dest_type in tag_mapping: if dest_tag <> 'N/A': if src_tag in tag_dict: if dest_type == 'datetime': tmp_timestamp = tag_dict[ src_tag].replace("T", " ").replace( "Z", "") if type(tmp_timestamp ) is not datetime: tmp_timestamp = datetime.strptime( tmp_timestamp, "%Y-%m-%d %H:%M:%S") row_dict[ dest_tag] = tmp_timestamp else: row_dict[dest_tag] = tag_dict[ src_tag] namedrow = settings.EntriesFields(**row_dict) rowsData.append(namedrow) rowNumber += 1 except Exception as e: print("crap") exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.info( "Exception processing row (%s): %s [%s / %s / %s]" % (e.message, file_fullpath, exc_type, fname, exc_tb.tb_lineno)) else: pass element.clear() xml_data.close() except Exception as e: print e.message print traceback.format_exc() pass