def main():
parser = utilities.get_argparser()
parser.add_argument("-obj_id",
help='The object ID of the file whose expired chunks are to be extracted',
type=int, dest="obj_id")
args = parser.parse_args()
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
objects = utilities.extract_objects(sorted_blocks)
chunk_offsets = []
for obj in objects:
for tag, chunk in obj.chunk_pairs:
obj_id = tag.object_id
if obj_id == args.obj_id:
is_expired = not tag.is_most_recent
is_data_chunk = not tag.isHeaderTag
if is_expired and is_data_chunk:
offset = chunk.offset
chunk_offsets.append(offset)
Write_chunks_to_file(args,chunk_offsets)
def main():
parser = utilities.get_argparser()
args = parser.parse_args()
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
nonerased_blocks = [b for b in sorted_blocks if not b.is_erased]
root, ext = os.path.splitext(args.imagefile)
root_path = os.path.dirname(args.imagefile)
out_path = os.path.join(root_path, root + '_chunks')
print out_path
if not os.path.exists(out_path):
os.mkdir(out_path)
else:
sys.stderr.write('Warning: chunks directory exists. Overwriting...\n')
for block in nonerased_blocks:
for tag, chunk in block.chunk_pairs:
filename = '%08d_04b%d_obj%04d_%03d.dd' % (chunk.offset,
tag.block_seq,
tag.object_id,
tag.chunk_id)
with open(os.path.join(out_path, filename), 'wb') as outfile:
outfile.write(chunk.get_bytes())
def main():
parser = utilities.get_argparser()
parser.add_argument("--object_id", dest='object_id', type=int, default=-1)
parser.add_argument("--object_name", dest='object_name', default='asd;lfjASDFeofijladksfjoe')
parser.add_argument("-v", dest='verbose', action="store_true")
args = parser.parse_args()
#read in and order all of the blocks, by reverse order of sequence number
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
objects_all = utilities.extract_objects(sorted_blocks)
#objects = [o for o in objects_all if o.object_id == args.object_id]
objects = []
#There might be multiple objects with the same ID due to deletions
for obj in objects_all:
# Deleted objects might have the name 'deleted'
# so we need to check past versions
name_set = set([x[0][1].name for x in obj.versions if 0 in x])
if obj.object_id == args.object_id or args.object_name in name_set:
objects.append(obj)
print 'Found %d object(s).' % len(objects)
for obj in objects:
summarize_object(obj, args.verbose)
def main():
parser = utilities.get_argparser()
args = parser.parse_args()
#read in and order all of the blocks, by reverse order of sequence number
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
objects = utilities.extract_objects(sorted_blocks)
current_objects = [o for o in objects if not o.is_deleted]
current_file_objects = []
for obj in current_objects:
#check the object type from the first header chunk.
if len(obj.versions) > 0 and obj.versions[0][0][1].obj_type == 1:
current_file_objects.append(obj)
print "object_id,name,chunk_id,count"
for obj in current_file_objects:
count = 0
name = obj.versions[0][0][1].name
for id in obj.chunkDict:
if id == 0:
continue
if len(obj.chunkDict[id]) > 1:
print "%d\t%s\t%d\t%d" % (obj.object_id, name, id, len(obj.chunkDict[id]))
def main():
parser = utilities.get_argparser()
args = parser.parse_args()
#read in and order all of the blocks, by reverse order of sequence number
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
objects = utilities.extract_objects(sorted_blocks)
print 'Extracted %d objects' % len(objects)
count = 0
for obj in objects:
for key in obj.chunkDict:
#Ignore header chunks, we don't really care why they changed.
if key == 0:
continue
#The number of old chunks
count += len(obj.chunkDict[key]) - 1
compare_chunks(obj.chunkDict[key])
print 'Found %d old chunks' % count
pass
def main():
parser = utilities.get_argparser()
args = parser.parse_args()
#read in and order all of the blocks, by reverse order of sequence number
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
objects = utilities.extract_objects(sorted_blocks)
current_objects = [o for o in objects if not o.is_deleted]
current_file_objects = []
for obj in current_objects:
#check the object type from the first header chunk.
if not obj.hasNoHeader and obj.versions[0][0][1].obj_type == 1:
current_file_objects.append(obj)
print "object_id,name,time_diff,expected,actual"
for obj in current_file_objects:
missing = get_missing_chunks_by_version(obj)
for expected, actual, time_diff, name in missing:
if time_diff < 0:
pass
print "%d\t%s\t%s\t%d\t%d" % (obj.object_id, name, time_diff, expected, actual)
def main():
parser = utilities.get_argparser()
parser.add_argument("--offsets",
nargs='*', dest="offsets")
args = parser.parse_args()
with open(args.imagefile, 'rb') as f:
for offset in args.offsets:
f.seek(int(offset))
sys.stdout.write(f.read(args.chunksize))
def main():
parser = utilities.get_argparser()
args = parser.parse_args()
#read in and order all of the blocks
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
missing_set = get_missing_block_numbers(sorted_blocks)
def main():
parser = utilities.get_argparser()
parser.add_argument("--write_to_file",
help='Write output to file in same directory as image.',
action='store_true', default=False, dest="write_to_file")
args = parser.parse_args()
if args.write_to_file:
print args.imagefile
blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
objects = utilities.extract_objects(blocks)
tuple_set = set()
for object in objects:
if object.hasNoHeader:
continue
for version in object.versions:
tuple_set.add((object.object_id, version[0][1].name))
root, ext = os.path.splitext(args.imagefile)
if args.write_to_file:
outfile = "%s_object_names.txt" % root
print 'Outfile: %s' % outfile
with open(outfile, 'w') as out:
for tuple in tuple_set:
out.write(str(tuple) + '\n')
else:
for tuple in tuple_set:
print tuple
def main():
parser = utilities.get_argparser()
args = parser.parse_args()
sorted_blocks = utilities.extract_ordered_blocks(args.imagefile,
args.chunksize,
args.oobsize,
args.blocksize,
args.tag_offset)
objects = utilities.extract_objects(sorted_blocks)
#current_objects = [o for o in objects if not o.is_deleted]
tmp_objects = objects[0:100]
total_bytes = 0
older_bytes = 0
non_header_chunk_versions = 0
total_chunk_versions = 0
for obj in objects:
for chunks in obj.chunkDict:
for version in obj.chunkDict[chunks]:
version_len = version[0].num_bytes
is_old_version = not version[0].is_most_recent
is_header_chunk = (version[0].chunk_id == 0)
if not is_header_chunk:
non_header_chunk_versions += 1
if is_old_version and (not is_header_chunk):
older_bytes += version_len
total_bytes += version_len
total_chunk_versions += 1
perc_old_bytes = older_bytes * 100.0 / total_bytes
print "Old chunk bytes that are not header chunks is %f percent" % perc_old_bytes
pass