def __init__(self, pointer): DiskScanner.__init__(self, pointer) self.found_file = set() self.parsed_file_review = {} self.found_indx = set() self.parsed_indx = {} self.indx_list = None self.found_boot = [] self.found_spc = []
def add_from_indx_allocation(self, parsed, part): """Add ghost entries to part from INDEX_ALLOCATION attributes in parsed. This procedure requires that the beginning of the partition has already been discovered.""" read_again = set() for attr in parsed['attributes']['$INDEX_ALLOCATION']: clusters_pos = 0 if 'runlist' not in attr: continue runlist = attr['runlist'] for entry in runlist: clusters_pos += entry['offset'] real_pos = clusters_pos * part.sec_per_clus + part.offset if real_pos in self.parsed_indx: content = self.parsed_indx[real_pos] # Check if the entry matches if parsed['record_n'] == content['parent']: discovered = set(c for c in content['children'] if c not in part.files) # If there are new files, read the INDX again if len(discovered): read_again.add(real_pos) img = DiskScanner.get_image(self) for position in read_again: dump = sectors(img, position, INDX_size) entries = parse_indx_record(dump)['entries'] self.add_indx_entries(entries, part)
def get_content(self, partition): """Extract the content of the file. This method works by extracting the $DATA attribute.""" if self.is_ghost: logging.error(u'Cannot restore ghost file {}'.format(self)) return None image = DiskScanner.get_image(partition.scanner) dump = sectors(image, File.get_offset(self), FILE_size) parsed = parse_file_record(dump) if not parsed['valid'] or 'attributes' not in parsed: logging.error(u'Invalid MFT entry for {}'.format(self)) return None attrs = parsed['attributes'] if ('$ATTRIBUTE_LIST' in attrs and partition.sec_per_clus is not None): _integrate_attribute_list(parsed, partition, image) if '$DATA' not in attrs: attrs['$DATA'] = [] datas = [d for d in attrs['$DATA'] if d['name'] == self.ads] if not len(datas): if not self.is_directory: logging.error(u'Cannot restore $DATA attribute(s) ' 'for {}'.format(self)) return None # TODO implemented compressed attributes for d in datas: if d['flags'] & 0x01: logging.error(u'Cannot restore compressed $DATA attribute(s) ' 'for {}'.format(self)) return None elif d['flags'] & 0x4000: logging.warning(u'Found encrypted $DATA attribute(s) ' 'for {}'.format(self)) # Handle resident file content if len(datas) == 1 and not datas[0]['non_resident']: single = datas[0] start = single['dump_offset'] + single['content_off'] end = start + single['content_size'] content = dump[start:end] return str(content) else: if partition.sec_per_clus is None: logging.error(u'Cannot restore non-resident $DATA ' 'attribute(s) for {}'.format(self)) return None non_resident = sorted( (d for d in attrs['$DATA'] if d['non_resident']), key=lambda x: x['start_VCN'] ) if len(non_resident) != len(datas): logging.warning( u'Found leftover resident $DATA attributes for ' '{}'.format(self) ) return self.content_iterator(partition, image, non_resident)
def add_from_indx_allocation(self, parsed, part): """Add ghost entries to part from INDEX_ALLOCATION attributes in parsed. This procedure requires that the beginning of the partition has already been discovered.""" read_again = set() for attr in parsed['attributes']['$INDEX_ALLOCATION']: clusters_pos = 0 if 'runlist' not in attr: continue runlist = attr['runlist'] for entry in runlist: clusters_pos += entry['offset'] real_pos = clusters_pos * part.sec_per_clus + part.offset if real_pos in self.parsed_indx: content = self.parsed_indx[real_pos] # Check if the entry matches if parsed['record_n'] == content['parent']: discovered = set( c for c in content['children'] if c not in part.files ) # If there are new files, read the INDX again if len(discovered): read_again.add(real_pos) img = DiskScanner.get_image(self) for position in read_again: dump = sectors(img, position, INDX_size) entries = parse_indx_record(dump)['entries'] self.add_indx_entries(entries, part)
def add_from_attribute_list(self, parsed, part, offset): """Add additional entries to part from attributes in ATTRIBUTE_LIST. Files with many attributes may have additional attributes not in the MFT entry. When this happens, it is necessary to find the other attributes. They may contain additional information, such as $DATA attributes for ADS. This procedure requires that the beginning of the partition has already been discovered.""" image = DiskScanner.get_image(self) _integrate_attribute_list(parsed, part, image) attrs = parsed['attributes'] if '$DATA' in attrs: for attribute in attrs['$DATA']: ads_name = attribute['name'] if ads_name and len(ads_name): part.add_file(NTFSFile(parsed, offset, ads=ads_name))
def add_from_mft_mirror(self, part): """Fix the first file records using the MFT mirror.""" img = DiskScanner.get_image(self) mirrpos = part.mftmirr_pos if mirrpos is None: return for i in xrange(4): node = part.get(i) if node is None or node.is_ghost: position = mirrpos + i * FILE_size dump = sectors(img, position, FILE_size) parsed = parse_file_record(dump) if parsed['valid'] and '$FILE_NAME' in parsed['attributes']: node = NTFSFile(parsed, position) part.add_file(node) logging.info( u'Repaired MFT entry #%s - %s in partition at offset ' '%s from backup', node.index, node.name, part.offset)
def add_from_mft_mirror(self, part): """Fix the first file records using the MFT mirror.""" img = DiskScanner.get_image(self) mirrpos = part.mftmirr_pos if mirrpos is None: return for i in xrange(4): node = part.get(i) if node is None or node.is_ghost: position = mirrpos + i * FILE_size dump = sectors(img, position, FILE_size) parsed = parse_file_record(dump) if parsed['valid'] and '$FILE_NAME' in parsed['attributes']: node = NTFSFile(parsed, position) part.add_file(node) logging.info( u'Repaired MFT entry #%s - %s in partition at offset ' '%s from backup', node.index, node.name, part.offset )
def get_partitions(self): """Get a list of the found partitions.""" partitioned_files = {} img = DiskScanner.get_image(self) logging.info('Parsing MFT entries') for position in self.found_file: dump = sectors(img, position, FILE_size) parsed = parse_file_record(dump) attrs = parsed.get('attributes', {}) if not parsed['valid'] or '$FILE_NAME' not in attrs: continue # Partition files based on corresponding entry 0 if parsed['record_n'] is not None: offset = position - parsed['record_n'] * FILE_size try: part = partitioned_files[offset] except KeyError: partitioned_files[offset] = NTFSPartition(self, offset) part = partitioned_files[offset] attributes = parsed['attributes'] if '$DATA' in attributes: for attribute in attributes['$DATA']: ads_name = attribute['name'] if ads_name: part.add_file( NTFSFile(parsed, position, ads=ads_name)) """Add the file again, just in case the $DATA attributes are missing.""" part.add_file(NTFSFile(parsed, position)) # Handle information deduced from INDX records if '$INDEX_ROOT' in attrs: self.add_from_indx_root(parsed, part) # Save for later use if '$INDEX_ALLOCATION' in attrs or '$ATTRIBUTE_LIST' in attrs: self.parsed_file_review[position] = parsed # TODO [Future] handle files for which there is no record_number # Parse INDX records logging.info('Parsing INDX records') for position in self.found_indx: dump = sectors(img, position, INDX_size) parsed = parse_indx_record(dump) if not parsed['valid']: continue entries = parsed['entries'] referred = (el['file_info']['parent_entry'] for el in entries) record_n = Counter(referred).most_common(1)[0][0] # Save references for future access self.parsed_indx[position] = { 'parent': record_n, 'children': set(el['record_n'] for el in entries) } indx_info = self.parsed_indx self.indx_list = SparseList( {pos: indx_info[pos]['parent'] for pos in indx_info}) # Extract boot record information logging.info('Reading boot sectors') for index in self.found_boot: dump = sectors(img, index, 1) parsed = unpack(dump, boot_sector_fmt) sec_per_clus = parsed['sectors_per_cluster'] self.found_spc.append(sec_per_clus) relative = parsed['MFT_addr'] * sec_per_clus mirr_relative = parsed['MFTmirr_addr'] * sec_per_clus part = None # Look for matching partition, either as boot sector or backup for delta in (0, parsed['sectors']): index = index - delta address = relative + index # Set partition as recoverable if address in partitioned_files: part = partitioned_files[address] part.set_recoverable(True) part.set_size(parsed['sectors']) part.offset = index part.sec_per_clus = sec_per_clus part.mftmirr_pos = mirr_relative + index break # Repair MFT if the mirror is available for address in list(partitioned_files): # This could have been deleted in a previous iteration if address not in partitioned_files: continue part = partitioned_files[address] mirrpos = part.mftmirr_pos if mirrpos is None: entry = part.get(1) # $MFTMirr if entry is None: continue else: # Infer MFT mirror position dump = sectors(img, entry.offset, FILE_size) mirror = parse_file_record(dump) if (mirror['valid'] and 'attributes' in mirror and '$DATA' in mirror['attributes']): datas = mirror['attributes']['$DATA'] if (len(datas) == 1 and datas[0]['non_resident'] and 'runlist' in datas[0] and len(datas[0]['runlist']) > 0 and 'offset' in datas[0]['runlist'][0]): relative = datas[0]['runlist'][0]['offset'] spc = part.sec_per_clus if spc is None: continue mirrpos = relative * spc + part.offset part.mftmirr_pos = mirrpos self.add_from_mft_mirror(part) # Remove bogus partitions generated by MFT mirrors if mirrpos in partitioned_files: bogus = partitioned_files[mirrpos] # Check if it looks like a MFT mirror if len(bogus.files) == 4 and max(bogus.files) < 4: logging.debug( 'Dropping bogus NTFS partition with MFT ' 'position %d generated by MFT mirror of ' 'partition at offset %d', bogus.mft_pos, part.offset) partitioned_files.pop(mirrpos) # Acquire additional information from $INDEX_ALLOCATION logging.info('Finding partition geometry') most_likely = self.most_likely_sec_per_clus() for address in partitioned_files: part = partitioned_files[address] if part.offset is None: # Find geometry by approximate string matching offset, sec_per_clus = self.find_boundary( part, address, most_likely) if offset is not None: part.set_recoverable(True) part.offset = offset part.sec_per_clus = sec_per_clus else: offset, sec_per_clus = part.offset, part.sec_per_clus if offset is not None: logging.info( 'Finalizing MFT reconstruction of partition at offset %i', offset) self.finalize_reconstruction(part) # Merge pieces from fragmented MFT for address in list(partitioned_files): # This could have been deleted in a previous iteration if address not in partitioned_files: continue part = partitioned_files[address] entry = part.get(0) # $MFT if entry is None or part.sec_per_clus is None: continue dump = sectors(img, entry.offset, FILE_size) parsed = parse_file_record(dump) if not parsed['valid'] or 'attributes' not in parsed: continue if '$ATTRIBUTE_LIST' in parsed['attributes']: _integrate_attribute_list(parsed, part, img) attrs = parsed['attributes'] if '$DATA' not in attrs or len(attrs['$DATA']) < 1: continue if 'runlist' not in attrs['$DATA'][0]: continue runlist = attrs['$DATA'][0]['runlist'] if len(runlist) > 1: logging.info( 'MFT for partition at offset %d is fragmented. Trying to ' 'merge %d parts...', part.offset, len(runlist)) clusters_pos = runlist[0]['offset'] spc = part.sec_per_clus size = runlist[0]['length'] for entry in runlist[1:]: clusters_pos += entry['offset'] real_pos = clusters_pos * part.sec_per_clus + part.offset position = real_pos - size * spc if position in partitioned_files: piece = partitioned_files[position] if piece.offset is None or piece.offset == part.offset: conflicts = [ i for i in piece.files if not piece.files[i].is_ghost and i in part.files and not part.files[i].is_ghost ] if not len(conflicts): logging.debug( 'Merging partition with MFT offset %d into' ' %s (fragmented MFT)', piece.mft_pos, part) # Merge the partitions merge(part, piece) # Remove the fragment partitioned_files.pop(position) else: logging.debug( 'NOT merging partition with MFT offset %d into' ' %s (possible fragmented MFT) due to conflicts', piece.mft_pos, part) size += entry['length'] return partitioned_files
def get_partitions(self): """Get a list of the found partitions.""" partitioned_files = {} img = DiskScanner.get_image(self) logging.info('Parsing MFT entries') for position in self.found_file: dump = sectors(img, position, FILE_size) try: parsed = parse_file_record(dump) except NotImplementedError: logging.error( 'Problem parsing record on sector %d', position ) continue attrs = parsed['attributes'] if 'attributes' in parsed else {} if not parsed['valid'] or '$FILE_NAME' not in attrs: continue # Partition files based on corresponding entry 0 if parsed['record_n'] is not None: offset = position - parsed['record_n'] * FILE_size try: part = partitioned_files[offset] except KeyError: partitioned_files[offset] = NTFSPartition(self, offset) part = partitioned_files[offset] attributes = parsed['attributes'] if '$DATA' in attributes: for attribute in attributes['$DATA']: ads_name = attribute['name'] if ads_name: part.add_file(NTFSFile(parsed, position, ads=ads_name)) """Add the file again, just in case the $DATA attributes are missing.""" part.add_file(NTFSFile(parsed, position)) # Handle information deduced from INDX records if '$INDEX_ROOT' in attrs: self.add_from_indx_root(parsed, part) # Save for later use if '$INDEX_ALLOCATION' in attrs or '$ATTRIBUTE_LIST' in attrs: self.parsed_file_review[position] = parsed # TODO [Future] handle files for which there is no record_number # Parse INDX records logging.info('Parsing INDX records') for position in self.found_indx: dump = sectors(img, position, INDX_size) parsed = parse_indx_record(dump) if not parsed['valid']: continue entries = parsed['entries'] referred = (el['file_info']['parent_entry'] for el in entries) record_n = Counter(referred).most_common(1)[0][0] # Save references for future access self.parsed_indx[position] = { 'parent': record_n, 'children': set(el['record_n'] for el in entries) } indx_info = self.parsed_indx self.indx_list = SparseList({ pos: indx_info[pos]['parent'] for pos in indx_info }) # Extract boot record information logging.info('Reading boot sectors') for index in self.found_boot: dump = sectors(img, index, 1) parsed = unpack(dump, boot_sector_fmt) sec_per_clus = parsed['sectors_per_cluster'] self.found_spc.append(sec_per_clus) relative = parsed['MFT_addr'] * sec_per_clus mirr_relative = parsed['MFTmirr_addr'] * sec_per_clus part = None # Look for matching partition, either as boot sector or backup for delta in (0, parsed['sectors']): index = index - delta address = relative + index # Set partition as recoverable if address in partitioned_files: part = partitioned_files[address] part.set_recoverable(True) part.set_size(parsed['sectors']) part.offset = index part.sec_per_clus = sec_per_clus part.mftmirr_pos = mirr_relative + index break # Repair MFT if the mirror is available for address in list(partitioned_files): # This could have been deleted in a previous iteration if address not in partitioned_files: continue part = partitioned_files[address] mirrpos = part.mftmirr_pos if mirrpos is None: entry = part.get(1) # $MFTMirr if entry is None: continue else: # Infer MFT mirror position dump = sectors(img, entry.offset, FILE_size) mirror = parse_file_record(dump) if (mirror['valid'] and 'attributes' in mirror and '$DATA' in mirror['attributes']): datas = mirror['attributes']['$DATA'] if (len(datas) == 1 and datas[0]['non_resident'] and 'runlist' in datas[0] and len(datas[0]['runlist']) > 0 and 'offset' in datas[0]['runlist'][0]): relative = datas[0]['runlist'][0]['offset'] spc = part.sec_per_clus if spc is None: continue mirrpos = relative * spc + part.offset part.mftmirr_pos = mirrpos self.add_from_mft_mirror(part) # Remove bogus partitions generated by MFT mirrors if mirrpos in partitioned_files: bogus = partitioned_files[mirrpos] # Check if it looks like a MFT mirror if len(bogus.files) == 4 and max(bogus.files) < 4: logging.debug( 'Dropping bogus NTFS partition with MFT ' 'position %d generated by MFT mirror of ' 'partition at offset %d', bogus.mft_pos, part.offset ) partitioned_files.pop(mirrpos) # Acquire additional information from $INDEX_ALLOCATION logging.info('Finding partition geometry') most_likely = self.most_likely_sec_per_clus() for address in partitioned_files: part = partitioned_files[address] if part.offset is None: # Find geometry by approximate string matching offset, sec_per_clus = self.find_boundary( part, address, most_likely ) if offset is not None: part.set_recoverable(True) part.offset = offset part.sec_per_clus = sec_per_clus else: offset, sec_per_clus = part.offset, part.sec_per_clus if offset is not None: logging.info( 'Finalizing MFT reconstruction of partition at offset %i', offset ) self.finalize_reconstruction(part) # Merge pieces from fragmented MFT for address in list(partitioned_files): # This could have been deleted in a previous iteration if address not in partitioned_files: continue part = partitioned_files[address] entry = part.get(0) # $MFT if entry is None or part.sec_per_clus is None: continue dump = sectors(img, entry.offset, FILE_size) parsed = parse_file_record(dump) if not parsed['valid'] or 'attributes' not in parsed: continue if '$ATTRIBUTE_LIST' in parsed['attributes']: _integrate_attribute_list(parsed, part, img) attrs = parsed['attributes'] if '$DATA' not in attrs or len(attrs['$DATA']) < 1: continue if 'runlist' not in attrs['$DATA'][0]: continue runlist = attrs['$DATA'][0]['runlist'] if len(runlist) > 1: logging.info( 'MFT for partition at offset %d is fragmented. Trying to ' 'merge %d parts...', part.offset, len(runlist) ) clusters_pos = runlist[0]['offset'] spc = part.sec_per_clus size = runlist[0]['length'] for entry in runlist[1:]: clusters_pos += entry['offset'] real_pos = clusters_pos * part.sec_per_clus + part.offset position = real_pos - size*spc if position in partitioned_files: piece = partitioned_files[position] if piece.offset is None or piece.offset == part.offset: conflicts = [ i for i in piece.files if not piece.files[i].is_ghost and i in part.files and not part.files[i].is_ghost ] if not len(conflicts): logging.debug( 'Merging partition with MFT offset %d into' ' %s (fragmented MFT)', piece.mft_pos, part ) # Merge the partitions merge(part, piece) # Remove the fragment partitioned_files.pop(position) else: logging.debug( 'NOT merging partition with MFT offset %d into' ' %s (possible fragmented MFT) due to conflicts', piece.mft_pos, part ) size += entry['length'] return partitioned_files