class SaveReader():
"""Class designed to retrieve basic data from files."""
def __init__(self, file_name):
self.file = open(file_name, 'r')
self.stream = ConstBitStream(self.file)
def __enter__(self):
return self
def __exit__(self, type_, value, traceback):
self.file.close()
def read_bytes(self, count):
"""Read a number of bytes and return a bitstring."""
return self.stream.read(count * 8)
def find_blocks(self):
"""Find all data blocks and return a tuple of their offsets."""
return tuple(self.stream.findall('0x40000000', bytealigned=True))
def read_int(self):
"""Read a 4 byte little endian int."""
return self.stream.read(32).intle
def read_ints(self, count):
"""Return a tuple of ints."""
return tuple(
map(lambda x: x.read(32).intle,
self.read_bytes(count * 4).cut(32)))
def read_string(self):
"""Read a string with the length given in the first 4 bytes."""
return self.stream.read('bytes:{}'.format(self.read_int())).decode(
"utf-8", 'replace')
def find_offsets():
s = ConstBitStream(filename=filename)
global patterns
pattern = patterns[patchloc]
find = list(
s.findall(pattern,
text_offset,
text_offset + text_size,
bytealigned=True)) # Only search within the .text segment
for off in find:
# Fix offset
off_fix = int(off / 8)
if off_fix < 0x10000 or off_fix > 0x3FFFC: # Limit range
continue
# Calculate instruction offset
global final
global hexval
final = off_fix - 4
hexval = '0x{0:0{1}X}'.format(
final, 6
) #change int back to uppercase hex (make sure we also print leading zero)
class FileReader:
""" Some basic functionality for reading data from Binary files. """
def __init__(self, filename):
self.filename = filename
if isinstance(filename, str):
self.file = open(filename, 'rb')
self.bits = ConstBitStream(open(filename, 'rb'))
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.file.close()
def skip_bytes(self, count=1):
self.bits.read('pad:{0}'.format(count * 8))
def peek_int(self):
return self.bits.peek(32).uintle
def forward_to_first_non_zero_byte(self, start, end):
self.bits.pos = start
while self.bits.pos < end and self.bits.read(8).intle == 0:
pass
self.bits.pos -= 8
def read_strings_from_block(self, start, end, stopAtEmptyString=False):
self.bits.pos = start
r = []
while self.bits.pos < end:
s = self.read_string()
if s == "" and stopAtEmptyString:
return r
r.append(s)
return tuple(r)
def read_int(self):
""" Read a single little endian 4 byte integer """
return self.bits.read(32).intle
def findall(self, bs):
return self.bits.findall(bs, bytealigned=True)
def read_bytes(self, count):
return self.bits.read(count * 8)
def read_byte(self, skip=0):
i = self.bits.read(8).uint
if skip > 0:
self.skip_bytes(skip)
return i
def read_string_safe(self):
return self.bits.read('bytes:{0}'.format(
self.read_byte(skip=3))).decode("utf-8", 'replace')
def find(self, bs, start, end):
return self.bits.find(bs, start, end, True)
def find_first(self, bs):
return self.bits.find(bs)
def extract_compressed_payloads(self):
files = []
occ = self.findall('0x789C')
i = 0
readSize = 2**12
for pos in occ:
self.bits.pos = pos
#read the start of the stream into a buffer.
if (self.bits.length - self.pos) < 8 * 2**12:
readSize = int((self.bits.length - self.pos) / 8)
buf = self.bits.read('bytes:{0}'.format(readSize))
zo = zlib.decompressobj()
#start the decompression
try:
stream = zo.decompress(buf)
except zlib.error: # right magic number but not a zlib stream.
continue
while zo.unused_data == b'' and readSize >= 2**12:
if (self.bits.length - self.pos) < 8 * 2**12:
readSize = int((self.bits.length - self.pos) / 8)
block = self.bits.read('bytes:{0}'.format(readSize))
if len(block) > 0:
try:
stream += zo.decompress(block)
except zlib.error:
pass
else:
break # we've reached EOF
with open(self.filename + '_' + str(i) + '.decompressed',
'wb') as fh:
fh.write(stream)
files.append(self.filename + '_' + str(i) + '.decompressed')
i += 1
return files
@property
def pos(self):
return self.bits.pos
@pos.setter
def pos(self, value):
self.bits.pos = value
def read_string(self):
""" Read an undelimited string with the length given in the first 4 bytes """
return self.bits.read('bytes:{0}'.format(self.read_int())).decode(
"utf-8", 'replace')
def zfs_search(search_file_path, pool_name, target_pool_name="kdiwlcik", buffer_size=buffer_size_default, print_starts_ends_only=False, starts=None, ends=None, tmp_dir="/tmp", loop_device=sp.check_output([losetup, "-f"]).strip(), ):
#if search_file_path != None:
# search_bytes = open(search_file_path, "r")
#else:
# search_bytes = sys.stdin
# logger.info("no input file specified, reading from stdin")
#raise RuntimeError("as long issue https://github.com/zfsonlinux/zfs/issues/2830 isn't fixed in the zpool command it doesn't make sense to use the script because copying files takes > 1 month regarding the number of combination, data volume and decreasing performance of a driver becoming fuller and fuller - on the other hand if it is fixed or a workaround found the script will be very efficient")
# check privileges first (this is polite)
if os.getuid() != 0:
raise RuntimeError("Privileges are necessary to invoke zpool import tests, exiting")
# validate loop_device option (assume that losetup -f always returns a
# valid and usable result)
if re.match("/dev/loop[0-9]+", loop_device) is None:
raise ValueError("loop_device '%s' is not a valid loop device specification" % (loop_device,))
# @TODO: more checks for loop_device adequacy are necessary
logger.info("using loop device %s" % (loop_device,))
if print_starts_ends_only and (starts != None or ends != None):
raise ValueError("starts or ends specified together with print-starts-ends-only which doesn't make sense (you don't want simply the list you specified on command line to be printed)")
if search_file_path is None:
raise ValueError("search_file_path mustn't be None")
if not os.path.exists(search_file_path):
raise ValueError("search_file_path '%s' doesn't exist" % (search_file_path,))
#if not os.path.isfile(search_file_path): # return false for device files
# raise ValueError("search_file_path '%s' isn't a file, but has to be" % (search_file_path,))
search_bytes = open(search_file_path, "r")
# Can initialise from files, bytes, etc.
if starts is None or ends is None:
s = ConstBitStream(search_bytes) # if argument name is omitted, auto is used
if starts is None:
bitstring_result = list(s.findall(zfs_start_bytes, bytealigned=True))
bytealigned_bitstring_result = [x/8 for x in bitstring_result]
starts = sorted(bytealigned_bitstring_result) # ConstBitStream.find returns the bit position (rather than byte position)
logger.info("found starts '%s'" % (str(starts)))
else:
starts = [int(x) for x in starts]
logger.info("using starts\n%s\nspecified on command line" % (str(starts)))
if ends is None:
bitstring_result = list(s.findall(zfs_end_bytes, bytealigned=True))
bytealigned_bitstring_result = [x/8 for x in bitstring_result]
ends = sorted(bytealigned_bitstring_result, reverse=True) # reverse causes testing from largest possible to smallest possible (this eventually causes I/O overhead, but possibility to find a pool in the largest set is much higher)
logger.info("found ends '%s'" % (str(ends)))
else:
ends = [int(x) for x in ends]
logger.info("using ends\n%s\nspecified on command line" % (str(ends)))
if print_starts_ends_only:
logger.info("found start points\n%s\n and end points\n%s\nExiting" % (str(starts), str(ends)))
return
logger.info("discarding start positions which don't have a corresponding start position after %s bytes" % (label_size, ))
valid_starts = []
for start in starts:
for start0 in starts:
if start0 == start + label_size:
valid_starts.append(start)
# do nothing with start0 because it might be valid for some strange reason, but if it is, it is more valid than start because there're always 2 ZFS pool labels
logger.info("The remaining valid starts are '%s'" % (str(valid_starts)))
starts = valid_starts
logger.info("discarding end positions which don't have a corresponding end position after %s bytes" % (label_size, ))
valid_ends = []
for end in ends:
for end0 in ends:
if end0 == end+ label_size:
valid_ends.append(end)
logger.info("The remaining valid ends are '%s'" % (str(valid_ends)))
ends = valid_ends
logger.info("trying all %s combinations of %s start and %s end points from largest to smallest eventual result" % (str(len(starts)*len(ends)), str(len(starts)), str(len(ends)), ))
for start in starts:
for end in ends:
if end < start:
logger.info("skipping occurance of end at '%s' before start at '%s'" % (str(end),str(start)))
continue
end = end + zfs_end_tail_bytes_count
sp.check_call([losetup, "-o", str(start), "--sizelimit", str(end-start), loop_device, search_file_path])
logger.info("mounting subset of search file %s from byte %s to byte %s under %s" % (search_file_path, str(start), str(end), loop_device))
# test zpool import
zpool_import_process = sp.Popen([zpool, "import", "-D", ], stdout=sp.PIPE, stderr=sp.PIPE) # there's no difference in returncode between no results and successful listing of possible import -> parse output
zpool_import_process.wait()
zpool_import_output_tuple = zpool_import_process.communicate()
zpool_import_output = zpool_import_output_tuple[1] # no pool available message is written onto stderr; subprocess.Popen.communicate can be invoked only once, second invokation causes error because performing I/O on closed file
zpool_import_output_stdout = zpool_import_output_tuple[0]
logger.debug("The output of the ''zpool import'' test command is:\n%s\n\n\n%s\n\n" % (zpool_import_output, zpool_import_output_stdout, ))
if zpool_import_process.returncode == 0 and (zpool_import_output is None or (zpool_import_output.strip() != "no pools available to import" and not "state: UNAVAIL" in zpool_import_output)) and (zpool_import_output_stdout is None or (not "state: UNAVAIL" in zpool_import_output and not "state: UNAVAIL" in zpool_import_output_stdout)): # zpool import returncodes are useless (finding a pool with status UNAVAIL returns 0)
logger.info("interval %s to %s possibly contains a valid zpool. The output of the ''zpool import'' test command is:\n%s\n\n\n%s\n\nSkipping further search" % (str(start), str(end), zpool_import_output, zpool_import_output_stdout))
logger.info("the loop device '%s' still provides the pool. Move the data from it to a reliable pool as soon as possible and detach the loop device yourself." % (loop_device,))
return
sp.check_call([losetup, "-d", loop_device]) # only detach loop device if import wasn't successful because otherwise the newly rescued pooled is displayed as faulted because the underlying loop device is no longer available
return
# Search to Start of Frame 0 code on byte boundary
foundMoviOffset = s.find('0x6D6F7669', bytealigned=True)
if foundMoviOffset:
moviOffset = foundMoviOffset[0] / 8
print("Found movi signature at byte offset %X" % moviOffset)
else:
print "Signature movi not found"
print ("Current bytepos %X" % s.bytepos)
foundIdx1Offset = s.find('0x69647831', bytealigned=True)
if foundIdx1Offset:
idx1Offset = foundIdx1Offset[0] / 8
print("Found idx1 signature at byte offset %X" % idx1Offset)
else:
print "Signature idx1 not found"
print ("Current bytepos %X" % s.bytepos)
found01dcOffset = s.findall('0x30316463', bytealigned=True)
for i in found01dcOffset:
i = (i /8) + 8
print("%X" % i)
s.bytepos = i
print s.bytepos
timeStamp = s.read(64).bin
print timeStamp
print ("Time is %d" % timeStamp)
# print filetime_to_dt(timeStamp)
# s.bytepos = i + 4
# print s.bytepos
# timelow = s.read(32).Q
# t = float(timehigh)*2**32 + timelow
# print (t*1e-7 - 11644473600)
class FileReader:
""" Some basic functionality for reading data from Binary files. """
def __init__(self, filename):
self.filename = filename
if isinstance(filename, str):
self.file = open(filename, 'rb')
self.bits = ConstBitStream(open(filename, 'rb'))
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.file.close()
def skip_bytes(self,count=1):
self.bits.read('pad:{0}'.format(count*8))
def peek_int(self):
return self.bits.peek(32).uintle
def forward_to_first_non_zero_byte(self, start, end):
self.bits.pos = start
while self.bits.pos < end and self.bits.read(8).intle == 0:
pass
self.bits.pos -= 8
def read_strings_from_block(self, start, end, stopAtEmptyString=False):
self.bits.pos = start
r = []
while self.bits.pos < end:
s = self.read_string()
if s == "" and stopAtEmptyString:
return r
r.append(s)
return tuple(r)
def read_int(self):
""" Read a single little endian 4 byte integer """
return self.bits.read(32).intle
def findall(self,bs):
return self.bits.findall(bs,bytealigned=True)
def read_bytes(self, count):
return self.bits.read(count*8)
def read_byte(self, skip = 0):
i = self.bits.read(8).uint
if skip > 0:
self.skip_bytes(skip)
return i
def read_string_safe(self):
return self.bits.read('bytes:{0}'.format(self.read_byte(skip=3))).decode("utf-8", 'replace')
def find(self, bs, start, end):
return self.bits.find(bs,start, end, True )
def find_first(self, bs):
return self.bits.find(bs)
def extract_compressed_payloads(self):
files = []
occ = self.findall('0x789C')
i = 0
readSize = 2**12
for pos in occ:
self.bits.pos = pos
#read the start of the stream into a buffer.
if (self.bits.length - self.pos) < 8*2**12:
readSize = int((self.bits.length - self.pos) / 8)
buf = self.bits.read('bytes:{0}'.format(readSize))
zo = zlib.decompressobj()
#start the decompression
try:
stream = zo.decompress(buf)
except zlib.error: # right magic number but not a zlib stream.
continue
while zo.unused_data == b'' and readSize >= 2**12:
if (self.bits.length - self.pos) < 8*2**12:
readSize = int((self.bits.length - self.pos) / 8)
block = self.bits.read('bytes:{0}'.format(readSize))
if len(block)> 0:
try:
stream += zo.decompress(block)
except zlib.error:
pass
else:
break # we've reached EOF
with open(self.filename + '_' + str(i) + '.decompressed', 'wb') as fh:
fh.write(stream)
files.append(self.filename + '_' + str(i) + '.decompressed')
i+=1
return files
@property
def pos(self):
return self.bits.pos
@pos.setter
def pos(self, value):
self.bits.pos = value
def read_string(self):
""" Read an undelimited string with the length given in the first 4 bytes """
return self.bits.read('bytes:{0}'.format(self.read_int())).decode("utf-8", 'replace')
def binary_search(pattern_file_path, search_file_path, match_flag,
match_file_path):
'''
Function for searching through binary files
'''
#bd_logger("Starting binary search...")
pattern_file = open(pattern_file_path, 'rb')
search_file = open(search_file_path, 'rb')
pattern_data = pattern_file.read()
search_file_size = os.path.getsize(search_file_path)
try:
s = ConstBitStream(search_file)
except:
bd_logger("Couldn't nmap an empty file! --> " + search_file_path)
return
occurances = s.findall(pattern_file, bytealigned=True)
occurances = list(occurances)
totalOccurances = len(occurances)
match_arr = []
for i in range(0, len(occurances)):
occuranceOffset_dec = int(occurances[i] / 8)
occuranceOffset_hex = hex(int(occurances[i] / 8))
print(
str(i + 1) + ') ' + 'Offset_hex: ' + str(occuranceOffset_hex) +
' Offset_dec: ' + str(occuranceOffset_dec) + " File_name: " +
str(search_file_path.split("\\")[-1]) + " File_size: " +
str(hex(search_file_size)) + " \\ " + str(search_file_size))
if match_flag == 1:
match_arr.append(occuranceOffset_dec)
pattern_file.close()
search_file.close()
if match_flag == 1:
match_file = open(match_file_path, "rt")
for match_entry in match_arr:
print("\n### " + "match_entry: " + str(match_entry) + " ###")
match_file.seek(0)
i = 0
for line in match_file:
#print(line)
i += 1
i_mem_address = int(
line.split(" file_offset=")[0].split("mem_address=")[-1])
i_file_offset = int(line.split(" file_offset=")[-1])
i_match_entry = int(match_entry)
if i_file_offset > i_match_entry:
#print("F_OFF: " + str(i_file_offset) )
break
j = 0
match_file.seek(0)
for line in match_file:
j += 1
if j == i - 1:
print("MATCHED LINE: " + line.rstrip("\n"))
i_mem_address = int(
line.split(" file_offset=")[0].split("mem_address=")
[-1])
i_file_offset = int(line.split(" file_offset=")[-1])
i_match_entry = int(match_entry)
i_off_diff = int(i_match_entry - i_file_offset)
i_real_mem_address = int(i_mem_address + i_off_diff)
print("REAL_MEM_ADRESS_dec=" + str(i_real_mem_address) +
" REAL_MEM_ADRESS_hex=" +
str(hex(i_real_mem_address)) +
" page_address_dec=" + str(i_mem_address) +
" page_address_hex=" + str(hex(i_mem_address)))
# Pull out the tempo, offset is number of BITS
s.pos = TEMPO_OFFSET
preciseBPM = s.read('uintle:24')
songTempo = preciseBPM / 10000
debugPrint("Tempo BPM is {} ({})".format(songTempo, hex(preciseBPM)))
# Pull out the time signature
s.pos = TIME_SIGNATURE_OFFSET
numerator = s.read('uintle:8')
denominator = s.read('uintle:8')
debugPrint("Time signature is {}/{}".format(numerator, 2**denominator))
# Generate an ordered list of offsets pointing to bits of the
# binary data that we are interested in
offset_list = list(s.findall('0x71537645', bytealigned=True)) #qSvE
offset_list.extend(list(s.findall('0x7165534D', bytealigned=True))) #qeSM
#offset_list.extend(list(s.findall('0x6B617254', bytealigned = True)))
#offset_list.extend(list(s.findall('0x74536e49', bytealigned = True)))
#offset_list.extend(list(s.findall('0x74537854', bytealigned = True)))
#offset_list.extend(list(s.findall('0x69766e45', bytealigned = True)))
sorted_offset_list = sorted(offset_list)
for thisOffset in sorted_offset_list:
s.pos = thisOffset
if bDebug:
debugPrint("Byte offset {}".format(thisOffset))
dumphex(64, s)
