def read(self, size):
buf = ''
leb = int(self.tell() / self._ubi.leb_size)
offset = self.tell() % self._ubi.leb_size
try:
if size < 0:
raise Exception('Bad Read Offset Request')
self._last_read_addr = self._ubi.blocks[self._blocks[leb]].file_offset + self._ubi.blocks[self._blocks[leb]].ec_hdr.data_offset + offset
except Exception as e:
error(self.read, 'Error', 'LEB: %s is corrupted or has no data.' % (leb))
raise Exception('Bad Read Offset Request')
verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size))
if leb == self._last_leb:
self.seek(self.tell() + size)
return self._last_buf[offset:offset+size]
else:
try:
buf = self._ubi.file.read_block_data(self._ubi.blocks[self._blocks[leb]])
self._last_buf = buf
self._last_leb = leb
self.seek(self.tell() + size)
return buf[offset:offset+size]
except Exception as e:
error(self, 'Fatal', 'read loc: %s, size: %s, LEB: %s, offset: %s, error: %s' % (self._last_read_addr, size, leb, offset, e))
def read(self, size):
buf = ''
leb = int(self.tell() / self._ubi.leb_size)
offset = self.tell() % self._ubi.leb_size
try:
if size < 0:
raise Exception('Bad Read Offset Request')
self._last_read_addr = self._ubi.blocks[self._blocks[leb]].file_offset + self._ubi.blocks[self._blocks[leb]].ec_hdr.data_offset + offset
except Exception as e:
error(self.read, 'Error', 'LEB: %s is corrupted or has no data.' % (leb))
raise Exception('Bad Read Offset Request')
verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size))
if leb == self._last_leb:
self.seek(self.tell() + size)
return self._last_buf[offset:offset+size]
else:
try:
buf = self._ubi.file.read_block_data(self._ubi.blocks[self._blocks[leb]])
self._last_buf = buf
self._last_leb = leb
self.seek(self.tell() + size)
return buf[offset:offset+size]
except Exception as e:
error(self, 'Fatal', 'read loc: %s, size: %s, LEB: %s, offset: %s, error: %s' % (self._last_read_addr, size, leb, offset, e))
def _process_reg_file(ubifs, inode, path):
try:
buf = ''
if 'data' in inode:
compr_type = 0
sorted_data = sorted(inode['data'], key=lambda x: x.key['khash'])
last_khash = sorted_data[0].key['khash'] - 1
for data in sorted_data:
# If data nodes are missing in sequence, fill in blanks
# with \x00 * UBIFS_BLOCK_SIZE
if data.key['khash'] - last_khash != 1:
while 1 != (data.key['khash'] - last_khash):
buf += '\x00' * UBIFS_BLOCK_SIZE
last_khash += 1
compr_type = data.compr_type
ubifs.file.seek(data.offset)
d = ubifs.file.read(data.compr_len)
buf += decompress(compr_type, data.size, d)
last_khash = data.key['khash']
verbose_log(
_process_reg_file,
'ino num: %s, compression: %s, path: %s' %
(inode['ino'].key['ino_num'], compr_type, path))
except Exception, e:
error(_process_reg_file, 'Warn',
'inode num:%s :%s' % (inode['ino'].key['ino_num'], e))
def _process_reg_file(ubifs, inode, path):
try:
buf = b''
if 'data' in inode:
compr_type = 0
sorted_data = sorted(inode['data'], key=lambda x: x.key['khash'])
last_khash = sorted_data[0].key['khash']-1
for data in sorted_data:
# If data nodes are missing in sequence, fill in blanks
# with \x00 * UBIFS_BLOCK_SIZE
if data.key['khash'] - last_khash != 1:
while 1 != (data.key['khash'] - last_khash):
buf += b'\x00'*UBIFS_BLOCK_SIZE
last_khash += 1
compr_type = data.compr_type
ubifs.file.seek(data.offset)
d = ubifs.file.read(data.compr_len)
buf += decompress(compr_type, data.size, d)
last_khash = data.key['khash']
verbose_log(_process_reg_file, 'ino num: %s, compression: %s, path: %s' % (inode['ino'].key['ino_num'], compr_type, path))
except Exception as e:
error(_process_reg_file, 'Warn', 'inode num:%s :%s' % (inode['ino'].key['ino_num'], e))
# Pad end of file with \x00 if needed.
if inode['ino'].size > len(buf):
buf += b'\x00' * (inode['ino'].size - len(buf))
return buf
def extract_blocks(ubi):
"""Get a list of UBI block objects from file
Arguments:.
Obj:ubi -- UBI object.
Returns:
Dict -- Of block objects keyed by PEB number.
"""
blocks = {}
ubi.file.seek(ubi.file.start_offset)
peb_count = 0
cur_offset = 0
# range instead of xrange, as xrange breaks > 4GB end_offset.
for i in range(ubi.file.start_offset, ubi.file.end_offset, ubi.file.block_size):
buf = ubi.file.read(ubi.file.block_size)
if buf.startswith(UBI_EC_HDR_MAGIC):
blk = description(buf)
blk.file_offset = i
blk.peb_num = ubi.first_peb_num + peb_count
blk.size = ubi.file.block_size
blocks[blk.peb_num] = blk
peb_count += 1
log(extract_blocks, blk)
verbose_log(extract_blocks, 'file addr: %s' % (ubi.file.last_read_addr()))
verbose_display(blk)
else:
cur_offset += ubi.file.block_size
ubi.first_peb_num = cur_offset/ubi.file.block_size
ubi.file.start_offset = cur_offset
return blocks
def read(self, size):
buf = ''
leb = int(self.tell() / self._ubi.leb_size)
offset = self.tell() % self._ubi.leb_size
self._last_read_addr = self._ubi.blocks[
self._blocks[leb]].file_offset + self._ubi.blocks[
self._blocks[leb]].ec_hdr.data_offset + offset
verbose_log(self,
'read loc: %s, size: %s' % (self._last_read_addr, size))
if leb == self._last_leb:
self.seek(self.tell() + size)
return self._last_buf[offset:offset + size]
else:
try:
buf = self._ubi.file.read_block_data(
self._ubi.blocks[self._blocks[leb]])
self._last_buf = buf
self._last_leb = leb
self.seek(self.tell() + size)
return buf[offset:offset + size]
except Exception, e:
error(
self, 'Fatal',
'read loc: %s, size: %s, LEB: %s, offset: %s, error: %s' %
(self._last_read_addr, size, leb, offset, e))
def read(self, size):
if self.end_offset < self.tell() + size:
error(self.read, 'Error', 'Block ends at %s which is greater than file size %s' % (self.tell() + size, self.end_offset))
raise Exception('Bad Read Offset Request')
self._last_read_addr = self.tell()
verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size))
return self._fhandle.read(size)
def read(self, size):
if self.end_offset < self.tell() + size:
error(self.read, 'Error', 'Block ends at %s which is greater than file size %s' % (self.tell() + size, self.end_offset))
raise Exception('Bad Read Offset Request')
self._last_read_addr = self.tell()
verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size))
return self._fhandle.read(size)
def extract_blocks(ubi):
"""Get a list of UBI block objects from file
Arguments:.
Obj:ubi -- UBI object.
Returns:
Dict -- Of block objects keyed by PEB number.
"""
blocks = {}
ubi.file.seek(ubi.file.start_offset)
peb_count = 0
cur_offset = 0
bad_blocks = []
# range instead of xrange, as xrange breaks > 4GB end_offset.
for i in range(ubi.file.start_offset, ubi.file.end_offset, ubi.file.block_size):
try:
buf = ubi.file.read(ubi.file.block_size)
except Exception as e:
if settings.warn_only_block_read_errors:
error(extract_blocks, 'Error', 'PEB: %s: %s' % (ubi.first_peb_num + peb_count, str(e)))
continue
else:
error(extract_blocks, 'Fatal', 'PEB: %s: %s' % (ubi.first_peb_num + peb_count, str(e)))
if buf.startswith(UBI_EC_HDR_MAGIC):
blk = description(buf)
blk.file_offset = i
blk.peb_num = ubi.first_peb_num + peb_count
blk.size = ubi.file.block_size
blocks[blk.peb_num] = blk
peb_count += 1
log(extract_blocks, blk)
verbose_log(extract_blocks, 'file addr: %s' % (ubi.file.last_read_addr()))
ec_hdr_errors = ''
vid_hdr_errors = ''
if blk.ec_hdr.errors:
ec_hdr_errors = ','.join(blk.ec_hdr.errors)
if blk.vid_hdr and blk.vid_hdr.errors:
vid_hdr_errors = ','.join(blk.vid_hdr.errors)
if ec_hdr_errors or vid_hdr_errors:
if blk.peb_num not in bad_blocks:
bad_blocks.append(blk.peb_num)
log(extract_blocks, 'PEB: %s has possible issue EC_HDR [%s], VID_HDR [%s]' % (blk.peb_num, ec_hdr_errors, vid_hdr_errors))
verbose_display(blk)
else:
cur_offset += ubi.file.block_size
ubi.first_peb_num = cur_offset/ubi.file.block_size
ubi.file.start_offset = cur_offset
return blocks
def _set_file_perms(path, inode):
#os.chmod(path, inode['ino'].mode)
#os.chown(path, inode['ino'].uid, inode['ino'].gid)
with open("perms.txt", 'a') as f:
f.write('perms:%o,owner:%s.%s,path:%s\n' %
(inode['ino'].mode, inode['ino'].uid, inode['ino'].gid, path))
verbose_log(
_set_file_perms, 'perms:%s, owner: %s.%s, path: %s' %
(inode['ino'].mode, inode['ino'].uid, inode['ino'].gid, path))
def read(self, size):
buf = ''
leb = int(self.tell() / self._ubi.leb_size)
offset = self.tell() % self._ubi.leb_size
self._last_read_addr = self._ubi.blocks[self._blocks[leb]].file_offset + self._ubi.blocks[self._blocks[leb]].ec_hdr.data_offset + offset
verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size))
if leb == self._last_leb:
self.seek(self.tell() + size)
return self._last_buf[offset:offset+size]
else:
try:
buf = self._ubi.file.read_block_data(self._ubi.blocks[self._blocks[leb]])
self._last_buf = buf
self._last_leb = leb
self.seek(self.tell() + size)
return buf[offset:offset+size]
except Exception, e:
error(self, 'Fatal', 'read loc: %s, size: %s, LEB: %s, offset: %s, error: %s' % (self._last_read_addr, size, leb, offset, e))
def extract_blocks(ubi):
"""Get a list of UBI block objects from file
Arguments:.
Obj:ubi -- UBI object.
Returns:
Dict -- Of block objects keyed by PEB number.
"""
blocks = {}
ubi.file.seek(ubi.file.start_offset)
peb_count = 0
cur_offset = 0
# range instead of xrange, as xrange breaks > 4GB end_offset.
for i in range(ubi.file.start_offset, ubi.file.end_offset,
ubi.file.block_size):
buf = ubi.file.read(ubi.file.block_size)
if buf.startswith(UBI_EC_HDR_MAGIC):
blk = description(buf)
blk.file_offset = i
blk.peb_num = ubi.first_peb_num + peb_count
blk.size = ubi.file.block_size
blocks[blk.peb_num] = blk
peb_count += 1
log(extract_blocks, blk)
verbose_log(extract_blocks,
'file addr: %s' % (ubi.file.last_read_addr()))
verbose_display(blk)
else:
cur_offset += ubi.file.block_size
ubi.first_peb_num = cur_offset / ubi.file.block_size
ubi.file.start_offset = cur_offset
return blocks
def _set_file_timestamps(path, inode):
os.utime(path, (inode['ino'].atime_sec, inode['ino'].mtime_sec))
verbose_log(_set_file_timestamps, 'timestamps: access: %s, modify: %s, path: %s' % (inode['ino'].atime_sec, inode['ino'].mtime_sec, path))
def read(self, size):
self._last_read_addr = self.tell()
verbose_log(self,
'read loc: %s, size: %s' % (self._last_read_addr, size))
return self._fhandle.read(size)
def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]):
"""Walk the index gathering Inode, Dir Entry, and File nodes.
Arguments:
Obj:ubifs -- UBIFS object.
Int:lnum -- Logical erase block number.
Int:offset -- Offset in logical erase block.
Dict:inodes -- Dict of ino/dent/file nodes keyed to inode number.
Returns:
Dict:inodes -- Dict of ino/dent/file nodes keyed to inode number.
'ino' -- Inode node.
'data' -- List of data nodes if present.
'dent' -- List of directory entry nodes if present.
"""
try:
if len(bad_blocks):
if lnum in bad_blocks:
return
ubifs.file.seek((ubifs.leb_size * lnum) + offset)
buf = ubifs.file.read(UBIFS_COMMON_HDR_SZ)
chdr = nodes.common_hdr(buf)
log(index , '%s file addr: %s' % (chdr, ubifs.file.last_read_addr()))
verbose_display(chdr)
node_buf = ubifs.file.read(chdr.len - UBIFS_COMMON_HDR_SZ)
file_offset = ubifs.file.last_read_addr()
except Exception as e:
if str(e) == 'Bad Read Offset Request' and settings.warn_only_block_read_errors:
bad_blocks.append(lnum)
return
else:
error(index, 'Fatal', 'LEB: %s, UBIFS offset: %s, error: %s' % (lnum, ((ubifs.leb_size * lnum) + offset), e))
if chdr.node_type == UBIFS_IDX_NODE:
try:
idxn = nodes.idx_node(node_buf)
except Exception as e:
if settings.warn_only_block_read_errors:
error(index, 'Error', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e))
return
else:
error(index, 'Fatal', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e))
log(index, '%s file addr: %s' % (idxn, file_offset))
verbose_display(idxn)
branch_idx = 0
for branch in idxn.branches:
verbose_log(index, '-------------------')
log(index, '%s file addr: %s' % (branch, file_offset + UBIFS_IDX_NODE_SZ + (branch_idx * UBIFS_BRANCH_SZ)))
verbose_display(branch)
index(ubifs, branch.lnum, branch.offs, inodes, bad_blocks)
branch_idx += 1
elif chdr.node_type == UBIFS_INO_NODE:
try:
inon = nodes.ino_node(node_buf)
except Exception as e:
if settings.warn_only_block_read_errors:
error(index, 'Error', 'Problem at file address: %s extracting ino_node: %s' % (file_offset, e))
return
else:
error(index, 'Fatal', 'Problem at file address: %s extracting ino_node: %s' % (file_offset, e))
ino_num = inon.key['ino_num']
log(index, '%s file addr: %s, ino num: %s' % (inon, file_offset, ino_num))
verbose_display(inon)
if not ino_num in inodes:
inodes[ino_num] = {}
inodes[ino_num]['ino'] = inon
elif chdr.node_type == UBIFS_DATA_NODE:
try:
datn = nodes.data_node(node_buf, (ubifs.leb_size * lnum) + UBIFS_COMMON_HDR_SZ + offset + UBIFS_DATA_NODE_SZ)
except Exception as e:
if settings.warn_only_block_read_errors:
error(index, 'Error', 'Problem at file address: %s extracting data_node: %s' % (file_offset, e))
return
else:
error(index, 'Fatal', 'Problem at file address: %s extracting data_node: %s' % (file_offset, e))
ino_num = datn.key['ino_num']
log(index, '%s file addr: %s, ino num: %s' % (datn, file_offset, ino_num))
verbose_display(datn)
if not ino_num in inodes:
inodes[ino_num] = {}
if not 'data' in inodes[ino_num]:
inodes[ino_num]['data']= []
inodes[ino_num]['data'].append(datn)
elif chdr.node_type == UBIFS_DENT_NODE:
try:
dn = nodes.dent_node(node_buf)
except Exception as e:
if settings.warn_only_block_read_errors:
error(index, 'Error', 'Problem at file address: %s extracting dent_node: %s' % (file_offset, e))
return
else:
error(index, 'Fatal', 'Problem at file address: %s extracting dent_node: %s' % (file_offset, e))
ino_num = dn.key['ino_num']
log(index, '%s file addr: %s, ino num: %s' % (dn, file_offset, ino_num))
verbose_display(dn)
if not ino_num in inodes:
inodes[ino_num] = {}
if not 'dent' in inodes[ino_num]:
inodes[ino_num]['dent']= []
inodes[ino_num]['dent'].append(dn)
def _set_file_timestamps(path, inode):
os.utime(path, (inode['ino'].atime_sec, inode['ino'].mtime_sec))
verbose_log(
_set_file_timestamps, 'timestamps: access: %s, modify: %s, path: %s' %
(inode['ino'].atime_sec, inode['ino'].mtime_sec, path))
def _set_file_perms(path, inode):
os.chown(path, inode['ino'].uid, inode['ino'].gid)
os.chmod(path, inode['ino'].mode)
verbose_log(
_set_file_perms, 'perms:%s, owner: %s.%s, path: %s' %
(inode['ino'].mode, inode['ino'].uid, inode['ino'].gid, path))
def read(self, size):
self._last_read_addr = self.tell()
verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size))
return self._fhandle.read(size)
def _set_file_perms(path, inode):
os.chmod(path, inode['ino'].mode)
os.chown(path, inode['ino'].uid, inode['ino'].gid)
verbose_log(_set_file_perms, 'perms:%s, owner: %s.%s, path: %s' % (inode['ino'].mode, inode['ino'].uid, inode['ino'].gid, path))
def index(ubifs, lnum, offset, inodes={}):
"""Walk the index gathering Inode, Dir Entry, and File nodes.
Arguments:
Obj:ubifs -- UBIFS object.
Int:lnum -- Logical erase block number.
Int:offset -- Offset in logical erase block.
Dict:inodes -- Dict of ino/dent/file nodes keyed to inode number.
Returns:
Dict:inodes -- Dict of ino/dent/file nodes keyed to inode number.
'ino' -- Inode node.
'data' -- List of data nodes if present.
'dent' -- List of directory entry nodes if present.
"""
try:
ubifs.file.seek((ubifs.leb_size * lnum) + offset)
buf = ubifs.file.read(UBIFS_COMMON_HDR_SZ)
chdr = nodes.common_hdr(buf)
log(index, '%s file addr: %s' % (chdr, ubifs.file.last_read_addr()))
verbose_display(chdr)
node_buf = ubifs.file.read(chdr.len - UBIFS_COMMON_HDR_SZ)
file_offset = ubifs.file.last_read_addr()
except Exception as e:
error(
index, 'Fatal', 'leb: %s, ubifs offset: %s, error: %s' %
(lnum, ((ubifs.leb_size * lnum) + offset), e))
if chdr.node_type == UBIFS_IDX_NODE:
idxn = nodes.idx_node(node_buf)
log(index, '%s file addr: %s' % (idxn, file_offset))
verbose_display(idxn)
branch_idx = 0
for branch in idxn.branches:
verbose_log(index, '-------------------')
log(
index,
'%s file addr: %s' % (branch, file_offset + UBIFS_IDX_NODE_SZ +
(branch_idx * UBIFS_BRANCH_SZ)))
verbose_display(branch)
index(ubifs, branch.lnum, branch.offs, inodes)
branch_idx += 1
elif chdr.node_type == UBIFS_INO_NODE:
inon = nodes.ino_node(node_buf)
ino_num = inon.key['ino_num']
log(index,
'%s file addr: %s, ino num: %s' % (inon, file_offset, ino_num))
verbose_display(inon)
if not ino_num in inodes:
inodes[ino_num] = {}
inodes[ino_num]['ino'] = inon
elif chdr.node_type == UBIFS_DATA_NODE:
datn = nodes.data_node(node_buf,
(ubifs.leb_size * lnum) + UBIFS_COMMON_HDR_SZ +
offset + UBIFS_DATA_NODE_SZ)
ino_num = datn.key['ino_num']
log(index,
'%s file addr: %s, ino num: %s' % (datn, file_offset, ino_num))
verbose_display(datn)
if not ino_num in inodes:
inodes[ino_num] = {}
if not 'data' in inodes[ino_num]:
inodes[ino_num]['data'] = []
inodes[ino_num]['data'].append(datn)
elif chdr.node_type == UBIFS_DENT_NODE:
dn = nodes.dent_node(node_buf)
ino_num = dn.key['ino_num']
log(index,
'%s file addr: %s, ino num: %s' % (dn, file_offset, ino_num))
verbose_display(dn)
if not ino_num in inodes:
inodes[ino_num] = {}
if not 'dent' in inodes[ino_num]:
inodes[ino_num]['dent'] = []
inodes[ino_num]['dent'].append(dn)
idxn = nodes.idx_node(node_buf)
except Exception as e:
if settings.warn_only_block_read_errors:
error(index, 'Error', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e))
return
else:
error(index, 'Fatal', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e))
log(index, '%s file addr: %s' % (idxn, file_offset))
verbose_display(idxn)
branch_idx = 0
for branch in idxn.branches:
verbose_log(index, '-------------------')
log(index, '%s file addr: %s' % (branch, file_offset + UBIFS_IDX_NODE_SZ + (branch_idx * UBIFS_BRANCH_SZ)))
verbose_display(branch)
index(ubifs, branch.lnum, branch.offs, inodes, bad_blocks)
branch_idx += 1
elif chdr.node_type == UBIFS_INO_NODE:
try:
inon = nodes.ino_node(node_buf)
except Exception as e:
if settings.warn_only_block_read_errors:
error(index, 'Error', 'Problem at file address: %s extracting ino_node: %s' % (file_offset, e))
return
else:
verbose_display(chdr)
node_buf = ubifs.file.read(chdr.len - UBIFS_COMMON_HDR_SZ)
file_offset = ubifs.file.last_read_addr()
except Exception, e:
error(index, 'Fatal', 'leb: %s, ubifs offset: %s, error: %s' % (lnum, ((ubifs.leb_size * lnum) + offset), e))
if chdr.node_type == UBIFS_IDX_NODE:
idxn = nodes.idx_node(node_buf)
log(index, '%s file addr: %s' % (idxn, file_offset))
verbose_display(idxn)
branch_idx = 0
for branch in idxn.branches:
verbose_log(index, '-------------------')
log(index, '%s file addr: %s' % (branch, file_offset + UBIFS_IDX_NODE_SZ + (branch_idx * UBIFS_BRANCH_SZ)))
verbose_display(branch)
index(ubifs, branch.lnum, branch.offs, inodes)
branch_idx += 1
elif chdr.node_type == UBIFS_INO_NODE:
inon = nodes.ino_node(node_buf)
ino_num = inon.key['ino_num']
log(index, '%s file addr: %s, ino num: %s' % (inon, file_offset, ino_num))
verbose_display(inon)
if not ino_num in inodes:
inodes[ino_num] = {}
