def main():
notify = util.Notifier()
total_count, total_size, bucket_size = upload_all(util.STAGING_PATH)
if total_count:
notify.send(
"uploaded %d files of total size %s; new bucket size is %s" %
(total_count, util.sizeof_fmt(total_size),
util.sizeof_fmt(bucket_size)))
else:
print("nothing to do")
def main():
# notify = util.Notifier()
start_time = time.time()
total_count, total_size, bucket_size = download_all(util.UNSTAGING_PATH)
total_time = time.time() - start_time
if total_count:
print(
"downloaded %d files of total size %s in %f seconds; total bucket size is %s"
% (total_count, util.sizeof_fmt(total_size), round(
total_time, 1), util.sizeof_fmt(bucket_size)))
else:
print("nothing to do")
def write(self, path):
if self.active:
self.data = np.array(self.data, dtype='bool') \
if self.name == 'y' or self.name == 'pf' else np.vstack(self.data)
print "Writing {:}, dtype={:}, size={:}".format(self.name, str(self.data.dtype),
util.sizeof_fmt(self.data.nbytes))
np.save(path + self.name, self.data)
def getFileAtUSN(self, filename, filerecord, filekey, usn, previousVersion=None, exactSize=True):
missing_pages = 0
decrypt_offset = 0
file_pages = []
logicalSize = filerecord.dataFork.logicalSize
for extent in self.volume.getAllExtents(filerecord.dataFork, filerecord.fileID):
for bn in xrange(extent.startBlock, extent.startBlock + extent.blockCount):
pn = self.pagesForLBN(bn).get(usn) #fail
if pn:
clear = self.decryptFileBlock(pn[-1], filekey, bn, decrypt_offset)
file_pages.append(clear)
elif previousVersion:
file_pages.append(previousVersion[len(file_pages)])
else:
file_pages.append(self.blankPage)
missing_pages += 1
decrypt_offset += self.pageSize
print "Recovered %d:%s %d missing pages, size %s, created %s, contentModDate %s" % \
(filerecord.fileID, filename.encode("utf-8"), missing_pages, sizeof_fmt(logicalSize), hfs_date(filerecord.createDate), hfs_date(filerecord.contentModDate))
filename = "%d_%d_%s" % (filerecord.fileID, usn, filename)
if missing_pages == 0:
filename = "OK_" + filename
self.okfiles += 1
data = "".join(file_pages)
if exactSize:
data = data[:logicalSize]
self.writeUndeletedFile(filename, data)
return file_pages
def dump_nand(self, filename):
f = open(filename, "wb")
self.send_command(CMD_DUMP)
zz = self.s.recv(8)
totalSize = struct.unpack("<Q", zz)[0]
recvSize = 0
print "Dumping %s NAND to %s" % (sizeof_fmt(totalSize), filename)
pbar = ProgressBar(totalSize)
pbar.start()
h = hashlib.sha1()
while recvSize < totalSize:
pbar.update(recvSize)
d = self.s.recv(8192 * 2)
if not d or len(d) == 0:
break
h.update(d)
f.write(d)
recvSize += len(d)
pbar.finish()
f.close()
print "NAND dump time : %s" % str(
datetime.timedelta(seconds=pbar.seconds_elapsed))
print "SHA1: %s" % h.hexdigest()
if recvSize != totalSize:
print "dump_nand FAIL"
def search(self, search_string, free_only=True, include_headers=True):
"""
Search the given string search_string on the Play Store.
search_string -- the string to search on the Play Store
free_only -- True if only costless apps should be searched for
include_headers -- True if the result table should show column names
"""
try:
results = self.api.search(search_string)
except IndexError:
results = []
if not results:
logger.info("No result")
return
all_results = []
if include_headers:
# Name of the columns
col_names = ["Title", "Creator", "Size", "Downloads", "Last Update", "AppID", "Version", "Rating"]
all_results.append(col_names)
# Compute results values
for doc in results:
for cluster in doc["child"]:
for app in cluster["child"]:
# skip that app if it not free
# or if it's beta (pre-registration)
if ('offer' not in app # beta apps (pre-registration)
or free_only
and app['offer'][0]['checkoutFlowRequired'] # not free to download
):
continue
details = app['details']['appDetails']
detail = [app['title'],
app['creator'],
util.sizeof_fmt(int(details['installationSize']))
if int(details['installationSize']) > 0 else 'N/A',
details['numDownloads'],
details['uploadDate'],
app['docid'],
details['versionCode'],
"%.2f" % app["aggregateRating"]["starRating"]
]
all_results.append(detail)
# Print a nice table
col_width = []
for column_indice in range(len(all_results[0])):
col_length = max([len("%s" % row[column_indice]) for row in all_results])
col_width.append(col_length + 2)
for result in all_results:
for indice, item in enumerate(result):
out = "".join(str(item).strip().ljust(col_width[indice]))
try:
print(out, end='')
except UnicodeEncodeError:
out = out.encode('utf-8', errors='replace')
print(out, end='')
print()
return all_results
def load_app_info_table(self):
self.info = {}
self.info["Application Name"] = self.apk.get_app_name()
self.info["Application Size"] = util.sizeof_fmt(
os.path.getsize(self.apk_path))
self.info["Android Version Name"] = self.apk.get_androidversion_name()
self.info["Android Version Code"] = self.apk.get_androidversion_code()
self.info["Android Package Name"] = self.apk.get_package()
self.info["Signature Name"] = self.apk.get_signature_name()
self.info["Uses Dynamic Code Loading"] = str(
analysis.is_dyn_code(self.x))
self.info["Uses Reflection"] = str(analysis.is_reflection_code(self.x))
self.info["Uses Crypto"] = str(analysis.is_crypto_code(self.x))
self.info["Privacy Leaks"] = str(len(self.get_privacy_leaks()))
self.info["Number of Providers"] = str(len(self.apk.get_providers()))
self.info["Number of Activities"] = str(len(self.apk.get_activities()))
self.info["Number of Services"] = str(len(self.apk.get_services()))
self.info["Number of Libraries"] = str(len(self.apk.get_libraries()))
self.info["Number of Permissions"] = str(
len(self.get_uses_permissions()))
self.info_actions = {}
self.info_actions["Application Name"] = None
self.info_actions["Application Size"] = None
self.info_actions["Android Version Name"] = None
self.info_actions["Android Version Code"] = None
self.info_actions["Android Package Name"] = None
self.info_actions["Signature Name"] = self.show_signature
self.info_actions["Uses Dynamic Code Loading"] = self.show_dyncode
self.info_actions["Uses Reflection"] = self.show_reflection
self.info_actions["Uses Crypto"] = self.show_cryptocode
self.info_actions["Privacy Leaks"] = self.show_privacy_leaks
self.info_actions["Number of Providers"] = self.show_providers
self.info_actions["Number of Activities"] = self.show_activities
self.info_actions["Number of Services"] = self.show_services
self.info_actions["Number of Libraries"] = self.show_libraries
self.info_actions["Number of Permissions"] = self.show_permissions
info_table = self.ui.appInfoTable
info_table.setRowCount(len(self.info))
info_table.setColumnWidth(1, 200)
info_table.horizontalHeader().setResizeMode(0,
QtGui.QHeaderView.Stretch)
row = 0
for key in sorted(self.info):
action = self.info_actions[key]
action_button = None
if action is not None:
action_button = QtGui.QPushButton()
action_button.setText("Show")
action_button.clicked.connect(action)
key_item = QtGui.QTableWidgetItem(key)
value_item = QtGui.QTableWidgetItem(self.info[key])
info_table.setItem(row, 0, key_item)
info_table.setItem(row, 1, value_item)
if action_button is not None:
info_table.setCellWidget(row, 2, action_button)
row += 1
def to_string(self) -> bool:
"""
print all value in a class
"""
print('Image size: {0}x{1}'.format(self.width, self.height))
print('-Total numbers: {}'.format(self.num))
print('-Total size: {}\n'.format(sizeof_fmt(self.total_size)))
return True
def AddCurrentItem(self, item):
if item not in self.DirList:
self.DirList.append(item)
self.lstSelectedDir.addItem(item)
self.TotalQty += count_files(item)
self.TotalSize += file_size(item)
self.sb.showMessage('Photo Selected: ' + str(self.TotalQty) +
' (' + sizeof_fmt(self.TotalSize) + ')')
else:
msg = "The directory '" + item + "' already exist into the list"
reply = QtGui.QMessageBox.information(self, 'Warning', msg, QtGui.QMessageBox.Ok)
def initGeometry(self, d):
self.metaSize = d.get("meta-per-logical-page", 0)
if self.metaSize == 0:
self.metaSize = 12
dumpedPageSize = d.get("dumpedPageSize", d["#page-bytes"] + self.metaSize + 8)
self.dump_size= d["#ce"] * d["#ce-blocks"] * d["#block-pages"] * dumpedPageSize
self.totalPages = d["#ce"] * d["#ce-blocks"] * d["#block-pages"]
nand_size = d["#ce"] * d["#ce-blocks"] * d["#block-pages"] * d["#page-bytes"]
hsize = sizeof_fmt(nand_size)
self.bfn = d.get("boot-from-nand", False)
self.ppn = d.get("ppn-device", False)
self.dumpedPageSize = dumpedPageSize
self.pageSize = d["#page-bytes"]
self.bootloaderBytes = d.get("#bootloader-bytes", 1536)
self.logicalPageSize = d.get("logical-page-size", self.pageSize)
self.emptyBootloaderPage = "\xFF" * self.bootloaderBytes
self.blankPage = "\xFF" * self.pageSize
self.nCEs =d["#ce"]
self.blocksPerCE = d["#ce-blocks"]
self.pagesPerBlock = d["#block-pages"]
self.pagesPerCE = self.blocksPerCE * self.pagesPerBlock
self.vendorType = d["vendor-type"]
self.deviceReadId = d.get("device-readid", 0)
self.banks_per_ce_vfl = d["banks-per-ce"]
if self.ppn:
self.slc_pages = d.get("slc-pages", 0)
self.block_bits = d.get("block-bits", 0)
self.cau_bits = d.get("cau-bits", 0)
self.page_bits = d.get("page-bits", 0)
if d.has_key("metadata-whitening"):
self.metadata_whitening = (d["metadata-whitening"].data == "\x01\x00\x00\x00")
if nand_chip_info.has_key(self.deviceReadId):
self.banks_per_ce_physical = nand_chip_info.get(self.deviceReadId)[7]
elif self.ppn:
self.banks_per_ce_physical = struct.unpack("<L", d["caus-ce"].data)[0]
else:
#raise Exception("Unknown deviceReadId %x" % self.deviceReadId)
print "!!! Unknown deviceReadId %x, assuming 1 physical bank /CE, will probably fail" % self.deviceReadId
self.banks_per_ce_physical = 1
print "Chip id 0x%x banks per CE physical %d" % (self.deviceReadId, self.banks_per_ce_physical)
self.blocks_per_bank = self.blocksPerCE / self.banks_per_ce_physical
if self.blocksPerCE & (self.blocksPerCE-1) == 0:
self.bank_address_space = self.blocks_per_bank
self.total_block_space = self.blocksPerCE
else:
bank_address_space = next_power_of_two(self.blocks_per_bank)
self.bank_address_space = bank_address_space
self.total_block_space = ((self.banks_per_ce_physical-1)*bank_address_space) + self.blocks_per_bank
self.bank_mask = int(math.log(self.bank_address_space * self.pagesPerBlock,2))
print "NAND geometry : %s (%d CEs (%d physical banks/CE) of %d blocks of %d pages of %d bytes data, %d bytes metdata)" % \
(hsize, self.nCEs, self.banks_per_ce_physical, self.blocksPerCE, self.pagesPerBlock, self.pageSize, d["meta-per-logical-page"])
def load_app_info_table(self):
self.info = {}
self.info["Application Name"] = self.apk.get_app_name()
self.info["Application Size"] = util.sizeof_fmt(os.path.getsize(self.apk_path))
self.info["Android Version Name"] = self.apk.get_androidversion_name()
self.info["Android Version Code"] = self.apk.get_androidversion_code()
self.info["Android Package Name"] = self.apk.get_package()
self.info["Signature Name"] = self.apk.get_signature_name()
self.info["Uses Dynamic Code Loading"] = str(analysis.is_dyn_code(self.x))
self.info["Uses Reflection"] = str(analysis.is_reflection_code(self.x))
self.info["Uses Crypto"] = str(analysis.is_crypto_code(self.x))
self.info["Privacy Leaks"] = str(len(self.get_privacy_leaks()))
self.info["Number of Providers"] = str(len(self.apk.get_providers()))
self.info["Number of Activities"] = str(len(self.apk.get_activities()))
self.info["Number of Services"] = str(len(self.apk.get_services()))
self.info["Number of Libraries"] = str(len(self.apk.get_libraries()))
self.info["Number of Permissions"] = str(len(self.get_uses_permissions()))
self.info_actions = {}
self.info_actions["Application Name"] = None
self.info_actions["Application Size"] = None
self.info_actions["Android Version Name"] = None
self.info_actions["Android Version Code"] = None
self.info_actions["Android Package Name"] = None
self.info_actions["Signature Name"] = self.show_signature
self.info_actions["Uses Dynamic Code Loading"] = self.show_dyncode
self.info_actions["Uses Reflection"] = self.show_reflection
self.info_actions["Uses Crypto"] = self.show_cryptocode
self.info_actions["Privacy Leaks"] = self.show_privacy_leaks
self.info_actions["Number of Providers"] = self.show_providers
self.info_actions["Number of Activities"] = self.show_activities
self.info_actions["Number of Services"] = self.show_services
self.info_actions["Number of Libraries"] = self.show_libraries
self.info_actions["Number of Permissions"] = self.show_permissions
info_table = self.ui.appInfoTable
info_table.setRowCount(len(self.info))
info_table.setColumnWidth(1, 200)
info_table.horizontalHeader().setResizeMode(0, QtGui.QHeaderView.Stretch)
row = 0
for key in sorted(self.info):
action = self.info_actions[key]
action_button = None
if action is not None:
action_button = QtGui.QPushButton()
action_button.setText("Show")
action_button.clicked.connect(action)
key_item = QtGui.QTableWidgetItem(key)
value_item = QtGui.QTableWidgetItem(self.info[key])
info_table.setItem(row,0,key_item)
info_table.setItem(row,1,value_item)
if action_button is not None:
info_table.setCellWidget(row,2,action_button)
row += 1
def main():
notify = util.Notifier()
errors, counts = check_incoming("/home/user/QubesIncoming",
util.STAGING_PATH)
lines = []
if errors:
lines += ["encountered %d errors:" % len(errors)]
for err in errors:
lines += [" * %s" % err]
if counts:
tcount = sum(count for count, size in counts.values())
tsize = sum(size for count, size in counts.values())
lines += [
"accepted %d files from %d VMs (%s)" %
(tcount, len(counts), util.sizeof_fmt(tsize))
]
for vm, (count, size) in sorted(counts.items()):
lines += [
" * %s: %d (%s)" % (repr(vm), count, util.sizeof_fmt(size))
]
if lines:
notify.send("\n".join(lines))
def pushButtonClrClk(self):
if self.lstSelectedDir.count() > 0:
reply = QtGui.QMessageBox.question(self,
'Clear List', "Are you sure to remove ALL items?",
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No,
QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
self.lstSelectedDir.clear()
self.DirList = []
self.TotalQty = 0
self.TotalSize = 0
self.sb.showMessage('Photo Selected: ' + str(self.TotalQty) +
' (' + sizeof_fmt(self.TotalSize) + ')')
def final_stats(self):
# Cave: the rates (data rate, frame rate) calculated here are an
# overestimate, in particular for short input videos. The reason is
# that we start keeping time only when we get the very first result,
# which means that we don't account for the decoding time of the
# first `nframes_per_process` frames.
# To get a better measure of decoding speed, use the multiprocreceiver
# with a high `repeat` parameter.
nbytes = util.sizeof_fmt(self.fragments_ok*64)
duration = time.time() - self.start
datarate = util.sizeof_fmt(self.fragments_ok*64./duration,
suffix='B/s')
framerate = self.framecount / duration
fmt = 'Decoded {nbytes} from {s.framecount} frames in {duration:.2f} s'
print fmt.format(s=self, nbytes=nbytes, duration=duration)
print 'Data rate: {}, frame rate: {:3.1f} frames/s'.format(datarate,
framerate)
if len(self.status_count) > 0:
print 'Status:',
print ', '.join('{}={}'.format(key, value)
for key, value in self.status_count.iteritems())
def pushButtonLeftClk(self):
if self.lstSelectedDir.currentItem() is not None:
reply = QtGui.QMessageBox.question(self, 'Remove Item',
"Are you sure to remove this item?",
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
current = self.lstSelectedDir.currentItem()
self.TotalQty -= count_files(current.text())
self.TotalSize -= file_size(current.text())
self.sb.showMessage('Photo Selected: ' + str(self.TotalQty) +
' (' + sizeof_fmt(self.TotalSize) + ')')
self.DirList.remove(current.text())
self.lstSelectedDir.takeItem(self.lstSelectedDir.row(current))
def final_stats(self):
# Cave: the rates (data rate, frame rate) calculated here are an
# overestimate, in particular for short input videos. The reason is
# that we start keeping time only when we get the very first result,
# which means that we don't account for the decoding time of the
# first `nframes_per_process` frames.
# To get a better measure of decoding speed, use the multiprocreceiver
# with a high `repeat` parameter.
nbytes = util.sizeof_fmt(self.fragments_ok * 64)
duration = time.time() - self.start
datarate = util.sizeof_fmt(self.fragments_ok * 64. / duration,
suffix='B/s')
framerate = self.framecount / duration
fmt = 'Decoded {nbytes} from {s.framecount} frames in {duration:.2f} s'
print fmt.format(s=self, nbytes=nbytes, duration=duration)
print 'Data rate: {}, frame rate: {:3.1f} frames/s'.format(
datarate, framerate)
if len(self.status_count) > 0:
print 'Status:',
print ', '.join('{}={}'.format(key, value)
for key, value in self.status_count.iteritems())
def do_ls(self, p):
dirDict = self.volume.ls((self.curdir + "/" + p).replace("//","/"))
if not dirDict:
return
for name in sorted(dirDict.keys()):
size = ""
protection_class = ""
record = dirDict[name]
if hasattr(record, "fileID"):
size = sizeof_fmt(record.dataFork.logicalSize)
cprotect = self.volume.getXattr(record.fileID, "com.apple.system.cprotect")
if cprotect:
protection_class = PROTECTION_CLASSES[struct.unpack("<L", cprotect[8:12])[0]]
print "%s\t%s\t%s\t%s" % (name[:30].ljust(30), size.ljust(10), hfs_date(record.createDate), protection_class)
def do_ptable(self, p):
pt = self.image.getPartitionTable()
print "Block device partition table"
print "".join(
map(lambda x: x.ljust(12),
["Index", "Name", "Start LBA", "End LBA", "Size"]))
for i in xrange(len(pt)):
p = pt[i]
print "".join(
map(lambda x: str(x).ljust(12), [
i, p.name, p.first_lba, p.last_lba,
sizeof_fmt(
(p.last_lba - p.first_lba) * self.image.pageSize)
]))
def _pretty_str(manifest):
strs = []
if hasattr(manifest, "project_root"):
strs.append(manifest.project_root)
else:
strs.append("(Unknown project root)")
if hasattr(manifest, "timestamp"):
strs.append("\tDate: %s" % manifest.timestamp.strftime("%c"))
if hasattr(manifest, "size"):
strs.append("\tSize: %s" % util.sizeof_fmt(manifest.size))
if hasattr(manifest, "git_branch"):
strs.append("\tBranch: %s" % manifest.git_branch)
if hasattr(manifest, "git_hash"):
strs.append("\tHash: %s" % manifest.git_hash)
return "\n".join(strs)
def _pretty_str(manifest):
strs = []
if hasattr(manifest, "project_root"):
strs.append(manifest.project_root)
else:
strs.append("(Unknown project root)")
if hasattr(manifest, "timestamp"):
strs.append("\tDate: %s" % manifest.timestamp.strftime("%c"))
if hasattr(manifest, "size"):
strs.append("\tSize: %s" % util.sizeof_fmt(manifest.size))
if hasattr(manifest, "git_branch"):
strs.append("\tBranch: %s" % manifest.git_branch)
if hasattr(manifest, "git_hash"):
strs.append("\tHash: %s" % manifest.git_hash)
return "\n".join(strs)
def json(self):
''' Return photo info in json
'''
json_photo = {
'id': self.id,
'path': self.path,
'title': self.title,
'caption': self.caption,
'size': sizeof_fmt(self.size),
'width': self.width,
'height': self.height,
'format': self.format,
'mode': self.mode,
'added_at': self.added_at,
'updated_at': self.updated_at,
'filename': self.filename()
}
return json_photo
def dumpToFile(self, outputfilename):
hs = sizeof_fmt((self.last_lba - self.lbaoffset) * self.pageSize)
print "Dumping partition to %s (%s)" % (outputfilename, hs)
flags = os.O_CREAT | os.O_RDWR
if sys.platform == "win32":
flags |= os.O_BINARY
fd=os.open(outputfilename, flags)
pbar = ProgressBar(self.last_lba - self.lbaoffset - 1)
pbar.start()
for i in xrange(self.lbaoffset, self.last_lba):
pbar.update(i-self.lbaoffset)
d = self.nand.readLPN(i, self.key)
if i == self.lbaoffset and d[0x400:0x402] != "HX":
print "FAIL? Not HFS partition or wrong key"
os.write(fd, d)
pbar.finish()
os.close(fd)
def dumpToFile(self, outputfilename):
hs = sizeof_fmt((self.last_lba - self.lbaoffset) * self.pageSize)
print("Dumping partition to %s (%s)" % (outputfilename, hs))
flags = os.O_CREAT | os.O_RDWR
if sys.platform == "win32":
flags |= os.O_BINARY
fd = os.open(outputfilename, flags)
pbar = ProgressBar(self.last_lba - self.lbaoffset - 1)
pbar.start()
for i in range(self.lbaoffset, self.last_lba):
pbar.update(i - self.lbaoffset)
d = self.nand.readLPN(i, self.key)
if i == self.lbaoffset and d[0x400:0x402] != "HX":
print("FAIL? Not HFS partition or wrong key")
os.write(fd, d)
pbar.finish()
os.close(fd)
def carveEMFVolumeJournal(volume):
journal = volume.readJournal()
print "Journal size : %s" % sizeof_fmt(len(journal))
hdr = journal_header.parse(journal)
sector_size = hdr.jhdr_size
nodeSize = volume.catalogTree.nodeSize
print "Collecting existing file ids"
fileIds = volume.listAllFileIds()
print "%d file IDs" % len(fileIds.keys())
files = {}
keys = {}
for i in xrange(0, len(journal), sector_size):
for k, v in carveBtreeNode(journal[i:i + nodeSize], HFSPlusCatalogKey,
HFSPlusCatalogData):
if v.recordType == kHFSPlusFileRecord:
name = getString(k)
h = hashlib.sha1(HFSPlusCatalogKey.build(k)).digest()
if files.has_key(h):
continue
if not fileIds.has_key(v.data.fileID):
#we only keep files where the first block is not marked as in use
if volume.isBlockInUse(
v.data.dataFork.HFSPlusExtentDescriptor[0].
startBlock) == False:
print "Found deleted file record", v.data.fileID, name
files[h] = (name, v)
for k, v in carveBtreeNode(journal[i:i + nodeSize], HFSPlusAttrKey,
HFSPlusAttrData):
if getString(k) == "com.apple.system.cprotect":
if not fileIds.has_key(k.fileID):
filekeys = keys.setdefault(k.fileID, [])
try:
cprotect = cprotect_xattr.parse(v.data)
except:
continue
#assert cprotect.xattr_major_version == 2
filekey = volume.keybag.unwrapKeyForClass(
cprotect.persistent_class, cprotect.persistent_key)
if filekey and not filekey in filekeys:
print "Found key for file", k.fileID
filekeys.append(filekey)
return files.values(), keys
def search(self, results_list, search_string, nb_results, free_only=True, include_headers=True):
try:
results = self.raw_search(results_list, search_string, nb_results)
except IndexError:
results = list()
if not results:
print("No result")
return
all_results = list()
if include_headers:
# Name of the columns
col_names = ["Title", "Creator", "Size", "Downloads", "Last Update", "AppID", "Version", "Rating"]
all_results.append(col_names)
# Compute results values
for result in results:
if free_only and result['offer'][0]['checkoutFlowRequired']: # if not Free to download
continue
l = [result['title'],
result['author'],
util.sizeof_fmt(result['installationSize']),
result['numDownloads'],
result['uploadDate'],
result['docId'],
result['versionCode'],
"%.2f" % result["aggregateRating"]["starRating"]
]
if len(all_results) < int(nb_results) + 1:
all_results.append(l)
if self.verbose:
# Print a nice table
col_width = list()
for column_indice in range(len(all_results[0])):
col_length = max([len("%s" % row[column_indice]) for row in all_results])
col_width.append(col_length + 2)
for result in all_results:
print("".join(str("%s" % item).strip().ljust(col_width[indice]) for indice, item in
enumerate(result)))
return all_results
def carveEMFVolumeJournal(volume):
journal = volume.readJournal()
print "Journal size : %s" % sizeof_fmt(len(journal))
hdr = journal_header.parse(journal)
sector_size = hdr.jhdr_size
nodeSize = volume.catalogTree.nodeSize
print "Collecting existing file ids"
fileIds = volume.listAllFileIds()
print "%d file IDs" % len(fileIds.keys())
files = {}
keys = {}
for i in xrange(0,len(journal),sector_size):
for k,v in carveBtreeNode(journal[i:i+nodeSize],HFSPlusCatalogKey, HFSPlusCatalogData):
if v.recordType == kHFSPlusFileRecord:
name = getString(k)
h = hashlib.sha1(HFSPlusCatalogKey.build(k)).digest()
if files.has_key(h):
continue
if not fileIds.has_key(v.data.fileID):
#we only keep files where the first block is not marked as in use
if volume.isBlockInUse(v.data.dataFork.HFSPlusExtentDescriptor[0].startBlock) == False:
print "Found deleted file record", v.data.fileID, name
files[h] = (name,v)
for k,v in carveBtreeNode(journal[i:i+nodeSize],HFSPlusAttrKey, HFSPlusAttrData):
if getString(k) == "com.apple.system.cprotect":
if not fileIds.has_key(k.fileID):
filekeys = keys.setdefault(k.fileID, [])
try:
cprotect = cprotect_xattr.parse(v.data)
except:
continue
#assert cprotect.xattr_major_version == 2
filekey = volume.keybag.unwrapKeyForClass(cprotect.persistent_class, cprotect.persistent_key)
if filekey and not filekey in filekeys:
print "Found key for file", k.fileID
filekeys.append(filekey)
return files.values(), keys
def download_all(local_path):
if not os.path.isdir(local_path):
raise Exception("nonexistent unstaging directory")
bucket = util.connect_bucket()
all_available = {obj.key: obj.size for obj in bucket.objects.all()}
remaining = set(all_available.keys())
for path, folders, files in os.walk(local_path):
for f in files:
lpath = os.path.join(path, f)
rpath = os.path.relpath(lpath, local_path)
local_size = os.stat(lpath).st_size
if rpath not in remaining:
raise RuntimeError(
"expected previously downloaded file %s of size %d to still exist remotely"
% (rpath, local_size))
if local_size != all_available[rpath]:
raise Exception(
"expected previously downloaded file %s to be of size %d, not %d"
% (rpath, all_available[rpath], local_size))
remaining.remove(rpath)
total_count, total_size = 0, 0
for rpath in remaining:
assert validate_path(rpath), "invalid path: %s" % repr(rpath)
lpath = os.path.join(local_path, rpath)
print("downloading", rpath, "to", lpath, "of size",
util.sizeof_fmt(all_available[rpath]))
if not os.path.isdir(os.path.dirname(lpath)):
os.makedirs(os.path.dirname(lpath))
bucket.download_file(rpath, lpath)
local_size = os.stat(lpath).st_size
if local_size != all_available[rpath]:
raise Exception(
"expected newly downloaded file %s to be of size %d, not %d" %
(rpath, all_available[rpath], local_size))
total_count += 1
total_size += local_size
bucket_size = sum(all_available.values())
return total_count, total_size, bucket_size
def getFileAtUSN(self,
filename,
filerecord,
filekey,
usn,
previousVersion=None,
exactSize=True):
missing_pages = 0
decrypt_offset = 0
file_pages = []
logicalSize = filerecord.dataFork.logicalSize
for extent in self.volume.getAllExtents(filerecord.dataFork,
filerecord.fileID):
for bn in xrange(extent.startBlock,
extent.startBlock + extent.blockCount):
pn = self.pagesForLBN(bn).get(usn) #fail
if pn:
clear = self.decryptFileBlock(pn[-1], filekey, bn,
decrypt_offset)
file_pages.append(clear)
elif previousVersion:
file_pages.append(previousVersion[len(file_pages)])
else:
file_pages.append(self.blankPage)
missing_pages += 1
decrypt_offset += self.pageSize
print "Recovered %d:%s %d missing pages, size %s, created %s, contentModDate %s" % \
(filerecord.fileID, filename.encode("utf-8"), missing_pages, sizeof_fmt(logicalSize), hfs_date(filerecord.createDate), hfs_date(filerecord.contentModDate))
filename = "%d_%d_%s" % (filerecord.fileID, usn, filename)
if missing_pages == 0:
filename = "OK_" + filename
self.okfiles += 1
data = "".join(file_pages)
if exactSize:
data = data[:logicalSize]
self.writeUndeletedFile(filename, data)
return file_pages
def dump_nand(self, filename):
f = open(filename, "wb")
self.send_command(CMD_DUMP)
zz = self.s.recv(8)
totalSize = struct.unpack("<Q", zz)[0]
recvSize = 0
print "Dumping %s NAND to %s" % (sizeof_fmt(totalSize), filename)
pbar = ProgressBar(totalSize)
pbar.start()
h = hashlib.sha1()
while recvSize < totalSize:
pbar.update(recvSize)
d = self.s.recv(8192*2)
if not d or len(d) == 0:
break
h.update(d)
f.write(d)
recvSize += len(d)
pbar.finish()
f.close()
print "NAND dump time : %s" % str(datetime.timedelta(seconds=pbar.seconds_elapsed))
print "SHA1: %s" % h.hexdigest()
if recvSize != totalSize:
print "dump_nand FAIL"
def do_FTP_LIST(self, netloc, path, user, passwd):
if not path.endswith('/'):
self.path += '/'
lst = []
table = '<table class="content"><thead><tr><th align="left">Content</th><th align="right">Size</th><th align="right">Modify</th></tr></thead><tbody>'
try:
ftp = ftplib.FTP(netloc)
ftp.login(user, passwd)
response = ftp.retrlines("LIST %s" % path, lst.append)
ftp.quit()
for line in lst:
self.logger.debug(line)
line_split = line.split(None, 8)
if line.startswith('d'):
line_split[8] += '/'
table += '<tr><td align="left"><a href="%s%s">%s</a></td><td align="right">%s</td><td align="right">%s %s %s</td></tr>\r\n' % (
self.path, urlquote(line_split[8]), line_split[8], line_split[4] if line.startswith('d') else sizeof_fmt(int(line_split[4])), line_split[5], line_split[6], line_split[7])
table += '<tr><td align="left">================</td><td align="right">==========</td><td align="right">=============</td></tr></tbody></table>\r\n'
table += '<p>%s</p>' % response
except Exception as e:
self.logger.warning("FTP Exception: %r" % e)
self.send_error(504, repr(e))
else:
msg = ['<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"><html>\n',
'<head><style type="text/css">.content tr{font-family:Consolas,"Droid Sans Mono", Menlo, Monospace;}</style></head>',
"<title>Directory listing for %s</title>\n" % path,
"<body>\n<h2>Directory listing for %s</h2>\n<hr>\n" % path,
table,
"<hr>\n</body>\n</html>\n"]
self.write(200, ''.join(msg), 'text/html')
def size_fmt(self):
return util.sizeof_fmt(self.size)
def get_file_size(self):
return util.sizeof_fmt(self.file_location)
def readFileHax(self, filename, filerecord, filekeys):
lba0 = self.first_lba + filerecord.dataFork.HFSPlusExtentDescriptor[
0].startBlock
filekey = None
good_usn = None
first_vpn = 0
first_usn = 0
hax = self.ftlhax
print "%d versions for first lba" % len(hax.get(lba0, []))
for k in filekeys:
for vpn in hax.get(lba0, []):
s, ciphertext = self.nand.ftl.YAFTL_readPage(vpn,
key=None,
lpn=None)
if not ciphertext:
continue
d = self.decryptFileBlock2(ciphertext, k, lba0, 0)
#hexdump(d[:16])
if isDecryptedCorrectly(d):
filekey = k
first_vpn = vpn
first_usn = good_usn = s.usn
block = vpn / self.nand.ftl.vfl.pages_per_sublk
break
if not filekey:
return False
logicalSize = filerecord.dataFork.logicalSize
missing_pages = 0
file_pages = []
lbns = []
for extent in self.volume.getAllExtents(filerecord.dataFork,
filerecord.fileID):
for bn in xrange(extent.startBlock,
extent.startBlock + extent.blockCount):
lbns.append(self.first_lba + bn)
datas = {}
usnblocksToLookAT = sorted(
filter(lambda x: x >= good_usn, self.userblocks.keys()))[:5]
print usnblocksToLookAT
usnblocksToLookAT.insert(0, 0)
first_block = True
done = False
for usn in usnblocksToLookAT:
if first_block:
bbtoc = self.getBBTOC(block)
first_block = False
else:
bbtoc = self.getBBTOC(self.userblocks[usn])
for lbn in bbtoc.keys():
if not lbn in lbns:
continue
idx = lbns.index(lbn)
s, ciphertext = self.nand.ftl.YAFTL_readPage(bbtoc[lbn],
key=None,
lpn=None)
if not ciphertext:
continue
ciphertext = self.decryptFileBlock2(ciphertext, filekey, lbn,
idx * self.pageSize)
if idx == 0:
if not isDecryptedCorrectly(ciphertext):
continue
datas[idx * self.pageSize] = (ciphertext, lbn - self.first_lba)
#if idx == len(lbns):
if len(datas) == len(lbns):
done = True
break
if done:
break
cleartext = ""
decrypt_offset = 0
for i in xrange(0, logicalSize, self.pageSize):
if datas.has_key(i):
ciphertext, lbn = datas[i]
cleartext += ciphertext
else:
cleartext += self.blankPage
missing_pages += 1
decrypt_offset += self.pageSize
print "Recovered %d:%s %d missing pages, size %s, created %s, contentModDate %s" % \
(filerecord.fileID, filename.encode("utf-8"), missing_pages, sizeof_fmt(logicalSize), hfs_date(filerecord.createDate), hfs_date(filerecord.contentModDate))
filename = "%d_%d_%s" % (filerecord.fileID, first_usn, filename)
if missing_pages == 0:
filename = "OK_" + filename
self.okfiles += 1
if True: #exactSize:
cleartext = cleartext[:logicalSize]
self.writeUndeletedFile(filename, cleartext)
return True
def readFileHax(self, filename, filerecord, filekeys):
lba0 = self.first_lba + filerecord.dataFork.HFSPlusExtentDescriptor[0].startBlock
filekey = None
good_usn = None
first_vpn = 0
first_usn = 0
hax = self.ftlhax
print "%d versions for first lba" % len(hax.get(lba0, []))
for k in filekeys:
for vpn in hax.get(lba0, []):
s, ciphertext = self.nand.ftl.YAFTL_readPage(vpn, key=None, lpn=None)
if not ciphertext:
continue
d = self.decryptFileBlock2(ciphertext, k, lba0, 0)
#hexdump(d[:16])
if isDecryptedCorrectly(d):
filekey = k
first_vpn = vpn
first_usn = good_usn = s.usn
block = vpn / self.nand.ftl.vfl.pages_per_sublk
break
if not filekey:
return False
logicalSize = filerecord.dataFork.logicalSize
missing_pages = 0
file_pages = []
lbns = []
for extent in self.volume.getAllExtents(filerecord.dataFork, filerecord.fileID):
for bn in xrange(extent.startBlock, extent.startBlock + extent.blockCount):
lbns.append(self.first_lba + bn)
datas = {}
usnblocksToLookAT = sorted(filter(lambda x: x >= good_usn, self.userblocks.keys()))[:5]
print usnblocksToLookAT
usnblocksToLookAT.insert(0, 0)
first_block = True
done = False
for usn in usnblocksToLookAT:
if first_block:
bbtoc = self.getBBTOC(block)
first_block = False
else:
bbtoc = self.getBBTOC(self.userblocks[usn])
for lbn in bbtoc.keys():
if not lbn in lbns:
continue
idx = lbns.index(lbn)
s, ciphertext = self.nand.ftl.YAFTL_readPage(bbtoc[lbn], key=None, lpn=None)
if not ciphertext:
continue
ciphertext = self.decryptFileBlock2(ciphertext, filekey, lbn, idx*self.pageSize)
if idx == 0:
if not isDecryptedCorrectly(ciphertext):
continue
datas[idx*self.pageSize] = (ciphertext, lbn - self.first_lba)
#if idx == len(lbns):
if len(datas) == len(lbns):
done=True
break
if done:
break
cleartext = ""
decrypt_offset = 0
for i in xrange(0,logicalSize, self.pageSize):
if datas.has_key(i):
ciphertext, lbn = datas[i]
cleartext += ciphertext
else:
cleartext += self.blankPage
missing_pages += 1
decrypt_offset += self.pageSize
print "Recovered %d:%s %d missing pages, size %s, created %s, contentModDate %s" % \
(filerecord.fileID, filename.encode("utf-8"), missing_pages, sizeof_fmt(logicalSize), hfs_date(filerecord.createDate), hfs_date(filerecord.contentModDate))
filename = "%d_%d_%s" % (filerecord.fileID, first_usn, filename)
if missing_pages == 0:
filename = "OK_" + filename
self.okfiles += 1
if True:#exactSize:
cleartext = cleartext[:logicalSize]
self.writeUndeletedFile(filename, cleartext)
return True
def do_ptable(self, p):
pt = self.image.getPartitionTable()
print "Block device partition table"
print "".join(map(lambda x:x.ljust(12), ["Index", "Name", "Start LBA", "End LBA", "Size"]))
for i in xrange(len(pt)):
p = pt[i]
print "".join(map(lambda x:str(x).ljust(12), [i, p.name, p.first_lba, p.last_lba, sizeof_fmt((p.last_lba - p.first_lba)*self.image.pageSize)]))
def readFileHax(self, filename, filerecord, filekeys):
lba0 = self.first_lba + filerecord.dataFork.HFSPlusExtentDescriptor[0].startBlock
filekey = None
good_usn = None
first_usn = 0
lba0_versions = self.nand.ftl.findAllVersions(lba0)
print "%d versions for first lba" % len(lba0_versions)
for k in filekeys:
for addr in lba0_versions:
ciphertext = self.nand.ftl.readPage1(addr, key=None, lpn=lba0)
if not ciphertext:
continue
d = self.decryptFileBlock2(ciphertext, k, lba0, 0)
if isDecryptedCorrectly(d):
hexdump(d[:16])
filekey = k
weaveSeq = addr[0]
break
if not filekey:
return False
logicalSize = filerecord.dataFork.logicalSize
missing_pages = 0
file_pages = []
lbns = []
for extent in self.volume.getAllExtents(filerecord.dataFork, filerecord.fileID):
for bn in xrange(extent.startBlock, extent.startBlock + extent.blockCount):
lbns.append(self.first_lba + bn)
datas = {}
first_block = True
done = False
for weaveSeq,lbn,ce,block,page in self.nand.ftl.findPagesInRange(weaveSeq, weaveSeq+50000):
if not lbn in lbns:
continue
idx = lbns.index(lbn)
ciphertext = self.nand.ftl.readPage1((weaveSeq,ce,block,page), key=None, lpn=lbn)
if not ciphertext:
continue
ciphertext = self.decryptFileBlock2(ciphertext, filekey, lbn, idx*self.pageSize)
if idx == 0:
if not isDecryptedCorrectly(ciphertext):
continue
datas[idx*self.pageSize] = (ciphertext, lbn - self.first_lba)
#if idx == len(lbns):
if len(datas) == len(lbns):
done=True
break
if done:
break
cleartext = ""
decrypt_offset = 0
for i in xrange(0,logicalSize, self.pageSize):
if datas.has_key(i):
ciphertext, lbn = datas[i]
cleartext += ciphertext
else:
cleartext += self.blankPage
missing_pages += 1
decrypt_offset += self.pageSize
print "Recovered %d:%s %d missing pages, size %s, created %s, contentModDate %s" % \
(filerecord.fileID, filename.encode("utf-8"), missing_pages, sizeof_fmt(logicalSize), hfs_date(filerecord.createDate), hfs_date(filerecord.contentModDate))
filename = "%d_%d_%s" % (filerecord.fileID, first_usn, filename)
if missing_pages == 0:
filename = "OK_" + filename
self.okfiles += 1
if True:#exactSize:
cleartext = cleartext[:logicalSize]
self.writeUndeletedFile(filename, cleartext)
return True
def main():
""" main CLI Entry """
parser = argparse.ArgumentParser()
# Add positional arguments
parser.add_argument('PATH',
nargs='+',
help='''Provides Source Directory(s) and Destination
Directory for image sorting. If --summary is
given, only needed Source Directory(s).''')
# Add optional arguments
parser.add_argument('-r',
'--recursive',
action='store_true',
help='get all images from subsequent directories')
parser.add_argument('-c',
'--copy',
action='store_true',
help='copy instead of move image files')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='print detail information')
parser.add_argument('-s',
'--summary',
action='store_true',
help='simulate the run, no file will be moved/copied')
parser.add_argument('-d',
'--dry-run',
action='store_true',
help='same as --summary option')
parser.add_argument('-i',
'--include',
action='store',
type=str,
help='''sorting only certain size indicated by
this option''')
parser.add_argument('-e',
'--exclude',
action='store',
type=str,
help='''exclude certain image size indicated by
this option''')
parser.add_argument('--unknown',
action='store_true',
help='sort all unknown/unreadable images into folder')
parser.add_argument('--unknownonly',
action='store_true',
help='sort only unknown/unreadable images only')
args = parser.parse_args()
# check error on arguments
_check_error(len(args.PATH), args.summary, (args.include, args.exclude),
(args.unknown, args.unknownonly))
# flag --summary if --dry-run is flaged
args.summary = bool(args.dry_run)
# Create destination directory if not exists
# print on-screen if verbose is true
if not args.summary:
util.create_dir(args.PATH[-1])
if args.verbose:
print('{}: is created.\n'.format(args.PATH[-1]))
# get the args.include or args.exclude value
limit_size: List[int] = []
if args.include:
limit_size = [int(num) for num in re.split('[x,]', args.include)]
elif args.exclude:
limit_size = [int(num) for num in re.split('[x,]', args.exclude)]
# Putting all boolean args into one bundle
bool_value: BoolCollection = BoolCollection(args.recursive, args.copy,
args.verbose, args.unknown,
bool(args.include))
# process image sorting functionality separately if --unknownonly option
# is on.
# it will process whethere --summary option is on or off
if args.unknownonly:
# src and dest will change depends on flag args.summary
src: str = args.PATH[:-1] if not args.summary else args.PATH
dest: str = args.PATH[-1] if not args.summary else ''
result: List[int] = sort_images.unknown_only([0, 0], src, dest,
args.summary, bool_value)
# print out info if --summary is flaged
if args.summary:
print('-Total numbers: {}'.format(result[0]))
print('-Total size: {}\n'.format(util.sizeof_fmt(result[1])))
# If summary arguments is true, no actual images is sorted
elif args.summary:
lst: List[ImagePtr] = []
lst = sort_images.summary(lst, args.PATH, bool_value, limit_size)
if not lst:
print('No image files found! Maybe using it with -r option?')
else:
print('\n===SUMMARY===')
for node in lst:
node.to_string()
else:
# if unknown is true, create unknown folder in destination before
# sorting
if args.unknown:
util.create_dir(os.path.join(args.destination, 'unknown'))
sort_images.sort_img(args.PATH[:-1], args.PATH[-1], bool_value,
limit_size)
#!/usr/bin/env python
import signac
from util import sizeof_fmt
project = signac.get_project()
counts = {'Coupon': 0}
sizes = {'Coupon': 0}
for job in project:
for fn in counts:
counts[fn] += job.doc.get(fn, {'shape': (0, 0)})['shape'][0]
for fn in sizes:
sizes[fn] += job.doc.get(fn, {'file_size': 0})['file_size']
print(counts)
for fn in sizes:
print(fn, sizeof_fmt(sizes[fn]))
