def test_all():
logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger(os.path.basename(__file__))
s0 = set()
v0 = Objects.VolumeObject()
v1 = Objects.VolumeObject()
s0.add(v0)
s0.add(v1)
_logger.debug("len(s0) = %r" % len(s0))
assert len(s0) == 2
f0 = Objects.FileObject()
f1 = Objects.FileObject()
f0.volume_object = v0
f1.volume_object = v0
s1 = set()
s1.add(f0.volume_object)
s1.add(f1.volume_object)
_logger.debug("len(s1) = %r" % len(s1))
assert len(s1) == 1
def test_disk_image_in_file_system():
dobj = Objects.DFXMLObject(version="1.2.0")
vobj = Objects.VolumeObject()
vobj.ftype_str = "iso9660"
dobj.append(vobj)
fobj_vobj = Objects.FileObject()
fobj_vobj.sha512 = TEST_HASH_1
vobj.append(fobj_vobj)
diobj = Objects.DiskImageObject()
vobj.append(diobj)
fobj_diobj = Objects.FileObject()
fobj_diobj.alloc_inode = False
fobj_diobj.alloc_name = False
fobj_diobj.sha512 = TEST_HASH_2
diobj.append(fobj_diobj)
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
vobj_reconst = dobj_reconst.volumes[0]
diobj_reconst = vobj_reconst.disk_images[0]
assert vobj_reconst.files[0].sha512 == TEST_HASH_1
assert diobj_reconst.files[0].sha512 == TEST_HASH_2
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def test_volume_error_roundtrip_with_file():
dobj = Objects.DFXMLObject(version="1.2.0")
vobj = Objects.VolumeObject()
dobj.append(vobj)
vobj.error = ERROR_STRING_V
assert vobj.error == ERROR_STRING_V
fobj = Objects.FileObject()
vobj.append(fobj)
fobj.error = ERROR_STRING_F
assert fobj.error == ERROR_STRING_F
assert vobj.error == ERROR_STRING_V
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
vobj_reconst = dobj_reconst.volumes[0]
fobj_reconst = vobj_reconst.files[0]
assert vobj_reconst.error == ERROR_STRING_V
assert fobj_reconst.error == ERROR_STRING_F
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def test_hfsplus_in_hfs():
dobj = Objects.DFXMLObject(version="1.2.0")
vobj_outer = Objects.VolumeObject()
vobj_outer.ftype_str = "hfs"
dobj.append(vobj_outer)
vobj_inner = Objects.VolumeObject()
vobj_inner.ftype_str = "hfsplus"
vobj_outer.append(vobj_inner)
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
vobj_outer_reconst = dobj_reconst.volumes[0]
vobj_inner_reconst = vobj_outer_reconst.volumes[0]
assert isinstance(vobj_inner_reconst, Objects.VolumeObject)
assert vobj_outer_reconst.ftype_str == "hfs"
assert vobj_inner_reconst.ftype_str == "hfsplus"
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def test_empty_object():
dobj = Objects.DFXMLObject(version="1.2.0")
vobj = Objects.VolumeObject()
dobj.append(vobj)
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
vobj_reconst = dobj_reconst.volumes[0]
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def test_all():
dobj = Objects.DFXMLObject(version="1.2.0")
# Make objects for simple appends.
diobj_0 = Objects.DiskImageObject()
psobj_0 = Objects.PartitionSystemObject()
pobj_0 = Objects.PartitionObject()
vobj_0 = Objects.VolumeObject()
vobj_0.ftype_str = "hfs"
fobj_0 = Objects.FileObject()
# Make objects for more exotic appends.
psobj_1 = Objects.PartitionSystemObject()
vobj_1 = Objects.VolumeObject()
vobj_1.ftype_str = "hfsplus"
fobj_dobj_1 = Objects.FileObject()
fobj_dobj_1.alloc_inode = False
fobj_dobj_1.alloc_name = False
fobj_psobj_1 = Objects.FileObject()
fobj_psobj_1.alloc_inode = False
fobj_psobj_1.alloc_name = False
fobj_pobj_1 = Objects.FileObject()
fobj_pobj_1.alloc_inode = False
fobj_pobj_1.alloc_name = False
# Do simple appends.
dobj.append(diobj_0)
diobj_0.append(psobj_0)
psobj_0.append(pobj_0)
pobj_0.append(vobj_0)
vobj_0.append(fobj_0)
# Do more exotic appends.
pobj_0.append(psobj_1)
vobj_0.append(vobj_1)
dobj.append(fobj_dobj_1)
psobj_0.append(fobj_psobj_1)
pobj_0.append(fobj_pobj_1)
def test_all():
logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger(os.path.basename(__file__))
v0 = Objects.VolumeObject()
v0.sector_size = 512
v0.block_size = 4096
v0.partition_offset = 32256
v0.ftype = -1
assert v0.ftype == -1
v0.ftype_str = 1
v0.block_count = 100000
v0.allocated_only = False
v0.first_block = 0
v0.last_block = v0.block_count
_logger.debug(repr(v0))
v1 = eval("Objects." + repr(v0))
e0 = v0.to_Element()
_logger.debug("e0 = %r" % e0)
v2 = Objects.VolumeObject()
v2.populate_from_Element(e0)
v1.block_size = 512
v2.partition_offset = v0.partition_offset + v0.block_count*v0.block_size
d01 = v0.compare_to_other(v1)
d02 = v0.compare_to_other(v2)
_logger.debug("d01 = %r" % d01)
assert d01 == set(["block_size"])
_logger.debug("d02 = %r" % d02)
assert d02 == set(["partition_offset"])
def test_volume_error_roundtrip_with_file_and_extns():
dobj = Objects.DFXMLObject(version="1.2.0")
vobj = Objects.VolumeObject()
dobj.append(vobj)
ET.register_namespace("testextra", XMLNS_TEST_EXTRA)
vobj.error = ERROR_STRING_V
# Dummy up a non-DFXML namespace element. This should be appendable.
e = ET.Element("{%s}extra_element" % XMLNS_TEST_EXTRA)
e.text = "Extra content"
vobj.externals.append(e)
# Dummy up a non-DFXML namespace 'error' element. This should be appendable.
e = ET.Element("{%s}error" % XMLNS_TEST_EXTRA)
e.text = "Extra error"
vobj.externals.append(e)
assert vobj.error == ERROR_STRING_V
fobj = Objects.FileObject()
vobj.append(fobj)
fobj.error = ERROR_STRING_F
assert fobj.error == ERROR_STRING_F
assert vobj.error == ERROR_STRING_V
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
vobj_reconst = dobj_reconst.volumes[0]
fobj_reconst = vobj_reconst.files[0]
assert vobj_reconst.error == ERROR_STRING_V
assert fobj_reconst.error == ERROR_STRING_F
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def test_error_after_file_system():
#TODO Bump version when feature branch merged into schema.
dobj = Objects.DFXMLObject(version="1.2.0+")
diobj = Objects.DiskImageObject()
dobj.append(diobj)
diobj.error = ERROR_1
vobj = Objects.VolumeObject()
vobj.error = ERROR_2
diobj.append(vobj)
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
diobj_reconst = dobj_reconst.disk_images[0]
vobj_reconst = diobj_reconst.volumes[0]
assert diobj_reconst.error == ERROR_1
assert vobj_reconst.error == ERROR_2
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def _test_file_in_non_fs_levels_deep(include_disk_image,
include_partition_system,
include_partition, include_file_system):
"""
This test follows a simple, vertical storage layer stack, but adds a file at each layer.
"""
dobj = Objects.DFXMLObject(version="1.2.0")
# Add file to top-level document.
fobj_dobj = Objects.FileObject()
fobj_dobj.alloc_inode = False
fobj_dobj.alloc_name = False
fobj_dobj.sha512 = TEST_HASH_1
dobj.append(fobj_dobj)
appender_stack = [dobj]
if include_disk_image:
# Add disk image to top-level document.
diobj = Objects.DiskImageObject()
appender_stack[-1].append(diobj)
appender_stack.append(diobj)
# Add file to disk image.
fobj_diobj = Objects.FileObject()
fobj_diobj.alloc_inode = False
fobj_diobj.alloc_name = False
fobj_diobj.sha512 = TEST_HASH_2
diobj.append(fobj_diobj)
if include_partition_system:
# Add partition system to disk image.
psobj = Objects.PartitionSystemObject()
appender_stack[-1].append(psobj)
appender_stack.append(psobj)
# Add file to partition system.
fobj_psobj = Objects.FileObject()
fobj_psobj.alloc_inode = False
fobj_psobj.alloc_name = False
fobj_psobj.sha512 = TEST_HASH_3
psobj.append(fobj_psobj)
if include_partition:
# Add partition to partition system, but not disk image.
if not (include_disk_image and not include_partition_system):
pobj = Objects.PartitionObject()
appender_stack[-1].append(pobj)
appender_stack.append(pobj)
# Add file to partition.
fobj_pobj = Objects.FileObject()
fobj_pobj.alloc_inode = False
fobj_pobj.alloc_name = False
fobj_pobj.sha512 = TEST_HASH_4
pobj.append(fobj_pobj)
if include_file_system:
# Add file system to anything but a partition system.
if not (include_partition_system and not include_partition):
vobj = Objects.VolumeObject()
appender_stack[-1].append(vobj)
appender_stack.append(vobj)
# Add file to file system.
fobj_vobj = Objects.FileObject()
fobj_vobj.sha512 = TEST_HASH_5
vobj.append(fobj_vobj)
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
container_stack = [dobj_reconst]
assert dobj_reconst.files[0].sha512 == TEST_HASH_1
if include_disk_image:
diobj_reconst = container_stack[-1].disk_images[0]
container_stack.append(diobj_reconst)
assert diobj_reconst.files[0].sha512 == TEST_HASH_2
if include_partition_system:
psobj_reconst = container_stack[-1].partition_systems[0]
container_stack.append(psobj_reconst)
assert psobj_reconst.files[0].sha512 == TEST_HASH_3
if include_partition:
if not (include_disk_image and not include_partition_system):
pobj_reconst = container_stack[-1].partitions[0]
container_stack.append(pobj_reconst)
assert pobj_reconst.files[0].sha512 == TEST_HASH_4
if include_file_system:
if not (include_partition_system and not include_partition):
vobj_reconst = container_stack[-1].volumes[0]
assert vobj_reconst.files[0].sha512 == TEST_HASH_5
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def _test_file_in_non_fs_levels_flat(include_disk_image,
include_partition_system,
include_partition, include_file_system):
"""
This test follows a simple, horizontal storage layer stack (every container attached to top document object), and adds a file for each container.
"""
dobj = Objects.DFXMLObject(version="1.2.0")
# Add file to top-level document.
fobj_dobj = Objects.FileObject()
fobj_dobj.alloc_inode = False
fobj_dobj.alloc_name = False
fobj_dobj.sha512 = TEST_HASH_1
dobj.append(fobj_dobj)
if include_disk_image:
# Add disk image.
diobj = Objects.DiskImageObject()
dobj.append(diobj)
# Add file to disk image.
fobj_diobj = Objects.FileObject()
fobj_diobj.alloc_inode = False
fobj_diobj.alloc_name = False
fobj_diobj.sha512 = TEST_HASH_2
diobj.append(fobj_diobj)
if include_partition_system:
# Add partition system.
psobj = Objects.PartitionSystemObject()
dobj.append(psobj)
# Add file to partition system.
fobj_psobj = Objects.FileObject()
fobj_psobj.alloc_inode = False
fobj_psobj.alloc_name = False
fobj_psobj.sha512 = TEST_HASH_3
psobj.append(fobj_psobj)
if include_partition:
# Add partition.
pobj = Objects.PartitionObject()
dobj.append(pobj)
# Add file to partition.
fobj_pobj = Objects.FileObject()
fobj_pobj.alloc_inode = False
fobj_pobj.alloc_name = False
fobj_pobj.sha512 = TEST_HASH_4
pobj.append(fobj_pobj)
if include_file_system:
# Add file system.
vobj = Objects.VolumeObject()
dobj.append(vobj)
# Add file to file system.
fobj_vobj = Objects.FileObject()
fobj_vobj.sha512 = TEST_HASH_5
vobj.append(fobj_vobj)
# Do file I/O round trip.
(tmp_filename, dobj_reconst) = libtest.file_round_trip_dfxmlobject(dobj)
try:
assert dobj_reconst.files[0].sha512 == TEST_HASH_1
if include_disk_image:
diobj_reconst = dobj_reconst.disk_images[0]
assert diobj_reconst.files[0].sha512 == TEST_HASH_2
if include_partition_system:
psobj_reconst = dobj_reconst.partition_systems[0]
assert psobj_reconst.files[0].sha512 == TEST_HASH_3
if include_partition:
pobj_reconst = dobj_reconst.partitions[0]
assert pobj_reconst.files[0].sha512 == TEST_HASH_4
if include_file_system:
vobj_reconst = dobj_reconst.volumes[0]
assert vobj_reconst.files[0].sha512 == TEST_HASH_5
except:
_logger.debug("tmp_filename = %r." % tmp_filename)
raise
os.remove(tmp_filename)
def main():
dobj = Objects.DFXMLObject(version="1.2.0")
dobj.program = sys.argv[0]
dobj.program_version = __version__
dobj.command_line = " ".join(sys.argv)
dobj.dc["type"] = "Example"
dobj.add_creator_library("Python", ".".join(map(str, sys.version_info[0:3]))) #A bit of a bend, but gets the major version information out.
dobj.add_creator_library("Objects.py", Objects.__version__)
dobj.add_creator_library("dfxml.py", Objects.dfxml.__version__)
vobj = Objects.VolumeObject()
dobj.append(vobj)
vobj.ftype_str = "examplefs"
# Define file system position.
vobj.byte_runs = Objects.ByteRuns()
vbr = Objects.ByteRun()
vobj.byte_runs.append(vbr)
vbr.img_offset = FILE_SYSTEM_START
vbr.len = DISK_IMAGE_SIZE - FILE_SYSTEM_START
fobj_specs = [
(
"first_sector.bin",
[
(0, 512)
]
),
(
"first_four_kilobytes.bin",
[
(0, 4000)
]
),
(
"contiguous_before_bad_region.dat",
[
(FILE_SYSTEM_START + 4096*1, 4096)
]
),
(
"contiguous_around_bad_region_left_edge.dat",
[
(DAMAGE_REGION_START - 4096, 8192)
]
),
(
"contiguous_in_bad_region.dat",
[
(DAMAGE_REGION_START + 4096*1, 4096)
]
),
(
"contiguous_around_bad_region_right_edge.dat",
[
(GOOD_REGION_START - 4096*1, 8192)
]
),
(
"contiguous_after_bad_region.dat",
[
(GOOD_REGION_START + 4096*2, 4096)
]
),
(
"fragmented_all_before_bad_region.dat",
[
(FILE_SYSTEM_START + 4096*10, 4096),
(FILE_SYSTEM_START + 4096*20, 4096),
(FILE_SYSTEM_START + 4096*30, 4096)
]
),
(
"fragmented_all_after_bad_region.dat",
[
(GOOD_REGION_START + 4096*10, 4096),
(GOOD_REGION_START + 4096*20, 4096),
(GOOD_REGION_START + 4096*30, 4096)
]
),
(
"fragmented_all_inside_bad_region.dat",
[
(DAMAGE_REGION_START + 4096*10, 4096),
(DAMAGE_REGION_START + 4096*20, 4096),
(DAMAGE_REGION_START + 4096*30, 4096)
]
),
(
"fragmented_beginning_inside_bad_region.dat",
[
(DAMAGE_REGION_START + 4096*40, 4096),
(GOOD_REGION_START + 4096*40, 4096)
]
),
(
"fragmented_middle_inside_bad_region.dat",
[
(FILE_SYSTEM_START + 4096*50, 4096),
(DAMAGE_REGION_START + 4096*50, 4096),
(GOOD_REGION_START + 4096*50, 4096)
]
),
(
"fragmented_end_inside_bad_region.dat",
[
(FILE_SYSTEM_START + 4096*60, 4096),
(DAMAGE_REGION_START + 4096*60, 4096)
]
),
(
"after_disk_image_end.dat",
[
(DISK_IMAGE_SIZE + 4096*1000, 4096)
]
),
(
"fragmented_partially_recoverable_directory",
[
(FILE_SYSTEM_START + 4096*170, 4096),
(DAMAGE_REGION_START + 4096*170, 4096),
(GOOD_REGION_START + 4096*170, 4096),
]
),
(
"fragmented_partially_recoverable_directory/child_file_1",
[
(FILE_SYSTEM_START + 4096*180, 4096)
]
),
(
"fragmented_partially_recoverable_directory/child_file_2",
[
(FILE_SYSTEM_START + 4096*190, 4096)
]
),
(
"fragmented_partially_recoverable_directory/child_file_3",
[
(FILE_SYSTEM_START + 4096*200, 4096)
]
),
(
"fragmented_partially_recoverable_directory/child_file_4",
[
(FILE_SYSTEM_START + 4096*210, 4096)
]
),
(
"fragmented_partially_recoverable_directory/child_file_9",
[
(GOOD_REGION_START + 4096*180, 4096)
]
)
]
for fobj_spec in fobj_specs:
fobj = Objects.FileObject()
vobj.append(fobj)
fobj.filename = fobj_spec[0]
fobj.alloc = True
# Naming convention for this sample - the .bin files are virtual files that reference a region outside of the file system.
if fobj.filename == "fragmented_partially_recoverable_directory":
fobj.name_type = "d"
elif fobj.filename.endswith(".bin"):
fobj.name_type = "v"
else:
fobj.name_type = "r"
fobj.data_brs = Objects.ByteRuns()
for interval in fobj_spec[1]:
br = Objects.ByteRun()
fobj.data_brs.append(br)
br.img_offset = interval[0]
br.len = interval[1]
fobj.filesize = sum([br.len for br in fobj.data_brs])
dobj.print_dfxml()
def main():
d = Objects.DFXMLObject(version="1.2.0")
d.program = sys.argv[0]
d.program_version = __version__
d.command_line = " ".join(sys.argv)
d.dc["type"] = "File system walk concatentation"
d.add_creator_library("Python", ".".join(
map(str, sys.version_info[0:3]
))) #A bit of a bend, but gets the major version information out.
d.add_creator_library("Objects.py", Objects.__version__)
d.add_creator_library("dfxml.py", Objects.dfxml.__version__)
_offsets_and_pxml_paths = []
for (lxfno, lxf) in enumerate(args.labeled_xml_file):
lxf_parts = lxf.split(":")
if len(lxf_parts) != 2 or not lxf_parts[0].isdigit():
raise ValueError(
"Malformed argument in labeled_xml_file. Expecting space-delimited list of '<number>:<path>'. This entry doesn't work: %r."
% lxf)
offset = int(lxf_parts[0])
path = lxf_parts[1]
_offsets_and_pxml_paths.append((offset, path))
offsets_and_pxml_paths = sorted(_offsets_and_pxml_paths)
for (pxml_path_index, (offset,
pxml_path)) in enumerate(offsets_and_pxml_paths):
_logger.debug("Running on path %r." % pxml_path)
pdo = Objects.parse(pxml_path)
building_volume = None
#Fetch or build volume we'll append
if len(pdo.volumes) > 1:
raise ValueError(
"An input DFXML document has multiple volumes; this script assumes each input document only has one. The document here has %d: %r."
% (len(pdo.volumes), pxml_path))
elif len(pdo.volumes) == 0:
v = Objects.VolumeObject()
building_volume = True
else:
v = pdo.volumes[0]
building_volume = False
v.partition_offset = offset
#Accumulate namespaces
for (prefix, url) in pdo.iter_namespaces():
d.add_namespace(prefix, url)
for obj in pdo:
#Force-update image offsets in byte runs
for brs_prop in ["data_brs", "name_brs", "inode_brs"]:
if hasattr(obj, brs_prop):
brs = getattr(obj, brs_prop)
if brs is None:
continue
for br in brs:
if not br.fs_offset is None:
br.img_offset = br.fs_offset + offset
#For files, set partition identifier and attach to partition
if isinstance(obj, Objects.FileObject):
obj.partition = pxml_path_index + 1
if building_volume:
v.append(obj)
#Collect the constructed and/or updated volume
d.append(v)
d.print_dfxml()
def test_all():
_logger = logging.getLogger(os.path.basename(__file__))
logging.basicConfig(level=logging.DEBUG)
XMLNS_TEST_CLAMSCAN = "file:///opt/local/bin/clamscan"
XMLNS_TEST_UNREGGED = "file:///dev/random"
ET.register_namespace("clam", XMLNS_TEST_CLAMSCAN)
vo = Objects.VolumeObject()
#Try and fail to add a non-Element to the list.
failed = None
_logger.debug("Before: " + repr(vo.externals))
try:
vo.externals.append(1)
failed = False
except TypeError:
failed = True
except:
failed = True
raise
_logger.debug("After: " + repr(vo.externals))
assert failed
failed = None
#Dummy up a non-DFXML namespace element. This should be appendable.
e = ET.Element("{%s}scan_results" % XMLNS_TEST_CLAMSCAN)
e.text = "Clean file system"
vo.externals.append(e)
#Dummy up a DFXML namespace element. This should not be appendable (the schema specifies other namespaces).
e = ET.Element("{%s}filename" % Objects.dfxml.XMLNS_DFXML)
e.text = "Superfluous name"
_logger.debug("Before: " + repr(vo.externals))
try:
vo.externals.append(e)
failed = False
except ValueError:
failed = True
except:
failed = True
raise
_logger.debug("After: " + repr(vo.externals))
assert failed
failed = None
#Add an element with the colon prefix style
e = ET.Element("clam:version")
e.text = "20140101"
vo.externals.append(e)
#Add an element that doesn't have an ET-registered namespace prefix.
e = ET.Element("{%s}test2" % XMLNS_TEST_UNREGGED)
e.text = "yes"
vo.externals.append(e)
#Test serialization
s = Objects._ET_tostring(vo.to_Element(
)) #TODO Maybe this should be more than an internal function.
_logger.debug(s)
if s.find("scan_results") == -1:
raise ValueError(
"Serialization did not output other-namespace element 'scan_results'."
)
if s.find("clam:version") == -1:
raise ValueError(
"Serialization did not output prefixed element 'clam:version'.")
if s.find("test2") == -1:
raise ValueError(
"Serialization did not output unregistered-prefix element 'test2'."
)
#Test de-serialization
vor = Objects.VolumeObject()
x = ET.XML(s)
vor.populate_from_Element(x)
_logger.debug("De-serialized: %r." % vor.externals)
assert len(vor.externals) == 3