def search():
try:
s = ConstBitStream(filename=kipname)
global find
global find2
find = s.find('0x003C00121F280071'
) # Atmosphere 11 and 12 (set for 1 byte patch)
find2 = s.find('0x01c0be121f00016b'
) # Atmosphere 13 > Current (set for 1 byte patch)
except OSError as e:
print("Unable to search files for this reason! %s" % e)
def search():
try:
from bitstring import ConstBitStream
s = ConstBitStream(filename=kipname)
global find
find = s.find(pattern)
global findnew
findnew = s.find(pattern2)
except OSError as e:
print("Error: %s : %s" % ("Search: ", e.strerror))
def find_fpt(data_bits: bitstring.ConstBitStream,
begin_at_offset_in_bytes: int = 0):
"""
Scan the bitstream looking for a Magic Number (see code below) that identifies a File Table.
Return the position of the file table (needed for further descent into the file structures) and some
other info (versions and number of entries)
:param data_bits: a BitStream object representing the input (generally constructed from the input specified to the
decoder, like a file. e.g. look at the function 'find_fpt_in_opened_file'
:param begin_at_offset_in_bytes: from where in the input the BitStream was constructed. This is NOT necessary for
correct operation; it is a convenience so that the return value will already have computed the location
relative to the start of the input file (but if the caller wishes, it can do this addition/offsetting itself,
and just leave the default 0 value for the parameter)
:return: a list of values: [0] = the location in the bitstream where the Magic Number for the File Table was found
(* the exact meaning of the "location" depends on begin_at_offset_in_bytes and the bitstream that was passed in;
see above)
[1] = the number of entries (code partition directories) in this file
[2] = the header version; this varies from CSME version to version
[3] = the version of the code p. directories. Again, this varies...
"""
f = data_bits.find('0x24465054', bytealigned=True
) # look for the string '$FPT' (expressed in hex, here)
if f:
# grab the number of entries, the header version and entry version, to return (this helps the human interpreting
# the result in figuring out if this is a valid FPT)
fpt_tag, num_entries, header_version, entry_version = data_bits.readlist(
'hex:32, uintle:32, uint:8, uint:8')
return [
begin_at_offset_in_bytes + int(f[0] / 8), num_entries,
header_version, entry_version
]
return None
def check_file(file_path, byte_array):
s = ConstBitStream(filename=file_path)
found = s.find(byte_array, bytealigned=True)
if found:
print("File: %s" % file_path)
print("Found start code at byte offset %d." % found[0])
s0f0, length, bitdepth, height, width = s.readlist('hex:16, uint:16, uint:8, 2*uint:16')
print("Width %d, Height %d\n" % (width, height))
def extractXMP(self, filename):
xmpStr = ""
# Can initialise from files, bytes, etc.
try:
s = ConstBitStream(filename = filename)
# Search for ":xmpmeta" string in file
keepSearching = True
while keepSearching:
keepSearching = False
colonXmpmetaInHexStr = '0x3a786d706d657461'
foundSt = s.find(colonXmpmetaInHexStr, bytealigned=True)
if foundSt:
byteStart = (int(foundSt[0])//8)
# The start of data can be "<xmp:xmpmeta" or "<x:xmpmeta"
s.bytepos = byteStart - 4
prevals = s.peeklist("4*uint:8")
prestr = ''.join(chr(i) for i in prevals)
# print (prestr, prestr[2:])
if prestr == "<xmp":
byteStart = byteStart - 4
prefix = "0x3c2f786d70" # "<\xmp" in hex
elif prestr[2:] == "<x":
byteStart = byteStart - 2
prefix = "0x3c2f78" # "<\x" in hex
else:
# print ("Cont")
keepSearching = True
continue
# print("Found start code at byte offset %d." % byteStart)
foundEnd = s.find(prefix + colonXmpmetaInHexStr, bytealigned=True)
if foundEnd:
byteEnd = (int(foundEnd[0])//8)
s.bytepos = byteStart
# print("Found end code at byte offset %d." % byteEnd)
xmpBytes = s.readlist(str(byteEnd-byteStart+len(prefix)//2+9) +"*uint:8")
xmpStr = ''.join(chr(i) for i in xmpBytes)
#if "Rating" in xmpStr:
# print (xmpStr)
except:
xmpStr = ""
return xmpStr
def _parse_request(bytes, path):
stream = ConstBitStream(bytes=bytes)
pos = stream.find(Bits(bytes=b'\r\n\r\n'), bytealigned=True)[0]
# For now I don't care about the request body.
header_bytes, _ = stream[:pos], stream[pos + 32:]
headers = dict()
header_lines = header_bytes.bytes.decode("ascii").split('\r\n')
for h in header_lines[1:]:
splits = h.split(":")
headers[splits[0]] = splits[1].strip()
return HttpRequest(headers, path, None)
def _parse_response(bytes):
stream = ConstBitStream(bytes=bytes)
pos = stream.find(Bits(bytes=b'\r\n\r\n'), bytealigned=True)[0]
header_bytes, body = stream[:pos], stream[pos + 32:]
headers = dict()
header_lines = header_bytes.bytes.decode("ascii").split('\r\n')
status_line = header_lines[0]
for h in header_lines[1:]:
splits = h.split(":")
headers[splits[0]] = splits[1].strip()
return HttpResponse(status_line, headers, body.tobytes())
def ff(file, mode='unknown'):
'''Read header from data file.'''
infile = open(file, 'r')
oldheader = infile.read(8)
# Finds "Dartmouth"
s = ConstBitStream(filename=file)
found = s.find('0x446172746d6f757468', bytealigned=True)
if found:
print("Found start code at byte offset %d." % found[0])
# s0f0, length, bitdepth, height, width = s.readlist('hex: 16, uint: 16,
# uint: 8, 2 * uint: 16')
# print("Width %d, Height %d" % (width, height))
else:
print("No way!!")
if oldheader[0:6] == "999999" and mode == 'unknown':
if oldheader[6:8] == "01":
mode = 'head'
elif oldheader[6:8] == "04":
mode = 'tail'
else:
print "ff(): Mode not defined in input or old header"
return(1)
elif oldheader[0:6] != "999999" and mode != 'tail':
print "ff(): No valid GGSE header signature."
return(1)
if mode == 'head':
header = infile.read(8184)
elif mode == 'tail':
infile.seek(-8192, os.SEEK_END)
header = infile.read(8192)
else:
print "ff(): Unknown mode."
return(1)
infile.close()
try:
fhead = fs(header)
except confp.ParsingError or confp.MissingSectionHeaderError:
print "ff(): error parsing file: No header in " + mode + " of file?"
fhead = 1
return fhead
def find_offset(of, in_data):
bitstream = ConstBitStream(in_data)
offset = bitstream.find(of, bytealigned=False)
return offset
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bitstring import ConstBitStream
import sys
c = ConstBitStream(filename=sys.argv[1])
#Most log entry ends with 0A, this helps us to seek to the beginning of an entry
start = c.find('0x0A', bytealigned=True)[0]
#Seek 8 byte into the stream to skip EOL from previus log entry
start += 8
c.pos = start
while True:
#Read and print the binary log header
header = c.readlist('8*uintle:32')
print header
#Get the size in bits of the log message
msgSize = (header[0] * 8 - 256)
#Move pointer after message
c.pos += msgSize
#Check if EOL is present after message, if true, move pointer 8 byte
if c.peek(8).hex == '0a':
c.pos += 8
class FileReader:
""" Some basic functionality for reading data from Binary files. """
def __init__(self, filename):
self.filename = filename
if isinstance(filename, str):
self.file = open(filename, 'rb')
self.bits = ConstBitStream(open(filename, 'rb'))
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.file.close()
def skip_bytes(self, count=1):
self.bits.read('pad:{0}'.format(count * 8))
def peek_int(self):
return self.bits.peek(32).uintle
def forward_to_first_non_zero_byte(self, start, end):
self.bits.pos = start
while self.bits.pos < end and self.bits.read(8).intle == 0:
pass
self.bits.pos -= 8
def read_strings_from_block(self, start, end, stopAtEmptyString=False):
self.bits.pos = start
r = []
while self.bits.pos < end:
s = self.read_string()
if s == "" and stopAtEmptyString:
return r
r.append(s)
return tuple(r)
def read_int(self):
""" Read a single little endian 4 byte integer """
return self.bits.read(32).intle
def findall(self, bs):
return self.bits.findall(bs, bytealigned=True)
def read_bytes(self, count):
return self.bits.read(count * 8)
def read_byte(self, skip=0):
i = self.bits.read(8).uint
if skip > 0:
self.skip_bytes(skip)
return i
def read_string_safe(self):
return self.bits.read('bytes:{0}'.format(
self.read_byte(skip=3))).decode("utf-8", 'replace')
def find(self, bs, start, end):
return self.bits.find(bs, start, end, True)
def find_first(self, bs):
return self.bits.find(bs)
def extract_compressed_payloads(self):
files = []
occ = self.findall('0x789C')
i = 0
readSize = 2**12
for pos in occ:
self.bits.pos = pos
#read the start of the stream into a buffer.
if (self.bits.length - self.pos) < 8 * 2**12:
readSize = int((self.bits.length - self.pos) / 8)
buf = self.bits.read('bytes:{0}'.format(readSize))
zo = zlib.decompressobj()
#start the decompression
try:
stream = zo.decompress(buf)
except zlib.error: # right magic number but not a zlib stream.
continue
while zo.unused_data == b'' and readSize >= 2**12:
if (self.bits.length - self.pos) < 8 * 2**12:
readSize = int((self.bits.length - self.pos) / 8)
block = self.bits.read('bytes:{0}'.format(readSize))
if len(block) > 0:
try:
stream += zo.decompress(block)
except zlib.error:
pass
else:
break # we've reached EOF
with open(self.filename + '_' + str(i) + '.decompressed',
'wb') as fh:
fh.write(stream)
files.append(self.filename + '_' + str(i) + '.decompressed')
i += 1
return files
@property
def pos(self):
return self.bits.pos
@pos.setter
def pos(self, value):
self.bits.pos = value
def read_string(self):
""" Read an undelimited string with the length given in the first 4 bytes """
return self.bits.read('bytes:{0}'.format(self.read_int())).decode(
"utf-8", 'replace')
print('Opened file ' + F)
print('Analyzing...')
s = ConstBitStream(filename=F)
print(s[32:96])
if s[32:96] != '0x6674797058415643':
print(
'No XAVC type tag detected. Please user original XAVC MP4 file. Exiting.'
)
sys.exit()
### Get filesize ###
filesize = os.path.getsize(F)
#check for mdat atom tag
sampl_check = s.find('0x6D646174000000', bytealigned=True)
if len(sampl_check) != 0:
#s.bytepos+=13
#sampl_string = s.read(4*8)
sampl_string = '0x001C0100'
else:
print(
'No mdat tags detected. Probably you have corrupted XAVC file. Exiting.'
)
sys.exit()
all_the_data = open(F, 'rb')
offset = (int(filesize / mmap.ALLOCATIONGRANULARITY) -
10) * mmap.ALLOCATIONGRANULARITY
m = mmap.mmap(all_the_data.fileno(),
__author__ = 'loki'
from bitstring import ConstBitStream, ReadError
import os
from filetimes import filetime_to_dt, dt_to_filetime, utc
s = ConstBitStream(filename='test3.avi')
statinfo = os.stat('test3.avi')
print statinfo.st_size
print ("Current bytepos %X" % s.bytepos)
# Search to Start of Frame 0 code on byte boundary
foundMoviOffset = s.find('0x6D6F7669', bytealigned=True)
if foundMoviOffset:
moviOffset = foundMoviOffset[0] / 8
print("Found movi signature at byte offset %X" % moviOffset)
else:
print "Signature movi not found"
print ("Current bytepos %X" % s.bytepos)
foundIdx1Offset = s.find('0x69647831', bytealigned=True)
if foundIdx1Offset:
idx1Offset = foundIdx1Offset[0] / 8
print("Found idx1 signature at byte offset %X" % idx1Offset)
else:
print "Signature idx1 not found"
print ("Current bytepos %X" % s.bytepos)
found01dcOffset = s.findall('0x30316463', bytealigned=True)
for i in found01dcOffset:
i = (i /8) + 8
print("%X" % i)
s.bytepos = i
print s.bytepos
timeStamp = s.read(64).bin
print timeStamp
filename = sys.argv[1]
if Path(filename).exists():
pass
else:
print(
'Something is wrong with your paths: \nUsage Example: python Loader-AutoIPS.py "package3"'
)
sys.exit(1)
loaderkip = "loader.kip"
startloc = 0
size = 0
end = 0
loader = ConstBitStream(filename=filename)
findloader = loader.find('0xCCCCCCCC4C6F61646572')
if sys.argv[2] == "":
workingdir = os.path.join(
os.path.dirname(os.path.abspath(os.path.realpath(sys.argv[0]))),
'output')
else:
workingdir = sys.argv[2]
Path(workingdir).mkdir(parents=True, exist_ok=True)
if Path(workingdir).exists():
pass
else:
print(
'Something is wrong with your paths: Usage Example: python Loader-AutoIPS.py "package3" "output"'
)
sys.exit(1)
info = ""
class FileReader:
""" Some basic functionality for reading data from Binary files. """
def __init__(self, filename):
self.filename = filename
if isinstance(filename, str):
self.file = open(filename, 'rb')
self.bits = ConstBitStream(open(filename, 'rb'))
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.file.close()
def skip_bytes(self,count=1):
self.bits.read('pad:{0}'.format(count*8))
def peek_int(self):
return self.bits.peek(32).uintle
def forward_to_first_non_zero_byte(self, start, end):
self.bits.pos = start
while self.bits.pos < end and self.bits.read(8).intle == 0:
pass
self.bits.pos -= 8
def read_strings_from_block(self, start, end, stopAtEmptyString=False):
self.bits.pos = start
r = []
while self.bits.pos < end:
s = self.read_string()
if s == "" and stopAtEmptyString:
return r
r.append(s)
return tuple(r)
def read_int(self):
""" Read a single little endian 4 byte integer """
return self.bits.read(32).intle
def findall(self,bs):
return self.bits.findall(bs,bytealigned=True)
def read_bytes(self, count):
return self.bits.read(count*8)
def read_byte(self, skip = 0):
i = self.bits.read(8).uint
if skip > 0:
self.skip_bytes(skip)
return i
def read_string_safe(self):
return self.bits.read('bytes:{0}'.format(self.read_byte(skip=3))).decode("utf-8", 'replace')
def find(self, bs, start, end):
return self.bits.find(bs,start, end, True )
def find_first(self, bs):
return self.bits.find(bs)
def extract_compressed_payloads(self):
files = []
occ = self.findall('0x789C')
i = 0
readSize = 2**12
for pos in occ:
self.bits.pos = pos
#read the start of the stream into a buffer.
if (self.bits.length - self.pos) < 8*2**12:
readSize = int((self.bits.length - self.pos) / 8)
buf = self.bits.read('bytes:{0}'.format(readSize))
zo = zlib.decompressobj()
#start the decompression
try:
stream = zo.decompress(buf)
except zlib.error: # right magic number but not a zlib stream.
continue
while zo.unused_data == b'' and readSize >= 2**12:
if (self.bits.length - self.pos) < 8*2**12:
readSize = int((self.bits.length - self.pos) / 8)
block = self.bits.read('bytes:{0}'.format(readSize))
if len(block)> 0:
try:
stream += zo.decompress(block)
except zlib.error:
pass
else:
break # we've reached EOF
with open(self.filename + '_' + str(i) + '.decompressed', 'wb') as fh:
fh.write(stream)
files.append(self.filename + '_' + str(i) + '.decompressed')
i+=1
return files
@property
def pos(self):
return self.bits.pos
@pos.setter
def pos(self, value):
self.bits.pos = value
def read_string(self):
""" Read an undelimited string with the length given in the first 4 bytes """
return self.bits.read('bytes:{0}'.format(self.read_int())).decode("utf-8", 'replace')
