def CreateZipSegment(self, filename):
self.MarkDirty()
segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)
# Is it in the cache?
res = self.resolver.CacheGet(segment_urn)
if res:
return res
self.resolver.Set(segment_urn, lexicon.AFF4_TYPE,
rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE))
self.resolver.Set(segment_urn, lexicon.AFF4_STORED, self.urn)
# Keep track of all the segments we issue.
self.children.add(segment_urn)
result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
result.LoadFromZipFile(self)
LOGGER.info("Creating ZipFileSegment %s",
result.urn.SerializeToString())
# Add the new object to the object cache.
return self.resolver.CachePut(result)
def CreateZipSegment(self, filename):
self.MarkDirty()
segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)
# Is it in the cache?
res = self.resolver.CacheGet(segment_urn)
if res:
return res
self.resolver.Set(
segment_urn, lexicon.AFF4_TYPE,
rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE))
self.resolver.Set(segment_urn, lexicon.AFF4_STORED, self.urn)
# Keep track of all the segments we issue.
self.children.add(segment_urn)
result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
result.LoadFromZipFile(self)
LOGGER.info("Creating ZipFileSegment %s",
result.urn.SerializeToString())
# Add the new object to the object cache.
return self.resolver.CachePut(result)
def OpenZipSegment(self, filename):
# Is it already in the cache?
segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)
if segment_urn not in self.members:
raise IOError("Segment %s does not exist yet" % filename)
res = self.resolver.CacheGet(segment_urn)
if res:
LOGGER.info("Openning ZipFileSegment (cached) %s", res.urn)
return res
result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
result.LoadFromZipFile(owner=self)
LOGGER.info("Openning ZipFileSegment %s", result.urn)
return self.resolver.CachePut(result)
def OpenZipSegment(self, filename):
if filename not in self.members:
raise IOError("Segment %s does not exist yet" % filename)
# Is it already in the cache?
segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)
res = self.resolver.CacheGet(segment_urn)
if res:
LOGGER.info("Openning ZipFileSegment (cached) %s", res.urn)
return res
result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
result.LoadFromZipFile(owner=self)
LOGGER.info("Openning ZipFileSegment %s", result.urn)
return self.resolver.CachePut(result)
def parse_cd(self, backing_store_urn):
with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
# Find the End of Central Directory Record - We read about 4k of
# data and scan for the header from the end, just in case there is
# an archive comment appended to the end.
backing_store.Seek(-BUFF_SIZE, 2)
ecd_real_offset = backing_store.Tell()
buffer = backing_store.Read(BUFF_SIZE)
end_cd, buffer_offset = EndCentralDirectory.FromBuffer(buffer)
urn_string = None
ecd_real_offset += buffer_offset
# Fetch the volume comment.
if end_cd.comment_len > 0:
backing_store.Seek(ecd_real_offset + end_cd.sizeof())
urn_string = backing_store.Read(end_cd.comment_len)
LOGGER.info("Loaded AFF4 volume URN %s from zip file.",
urn_string)
#if end_cd.size_of_cd == 0xFFFFFFFF:
# end_cd, buffer_offset = Zip64EndCD.FromBuffer(buffer)
#LOGGER.info("Found ECD at %#x", ecd_real_offset)
# There is a catch 22 here - before we parse the ZipFile we dont
# know the Volume's URN, but we need to know the URN so the
# AFF4FactoryOpen() can open it. Therefore we start with a random
# URN and then create a new ZipFile volume. After parsing the
# central directory we discover our URN and therefore we can delete
# the old, randomly selected URN.
if urn_string and self.urn != urn_string:
self.resolver.DeleteSubject(self.urn)
self.urn.Set(utils.SmartUnicode(urn_string))
# Set these triples so we know how to open the zip file again.
self.resolver.Set(self.urn, lexicon.AFF4_TYPE,
rdfvalue.URN(lexicon.AFF4_ZIP_TYPE))
self.resolver.Set(self.urn, lexicon.AFF4_STORED,
rdfvalue.URN(backing_store_urn))
self.resolver.Set(backing_store_urn, lexicon.AFF4_CONTAINS,
self.urn)
directory_offset = end_cd.offset_of_cd
directory_number_of_entries = end_cd.total_entries_in_cd
# Traditional zip file - non 64 bit.
if directory_offset > 0 and directory_offset != 0xffffffff:
# The global difference between the zip file offsets and real
# file offsets. This is non zero when the zip file was appended
# to another file.
self.global_offset = (
# Real ECD offset.
ecd_real_offset - end_cd.size_of_cd -
# Claimed CD offset.
directory_offset)
LOGGER.info("Global offset: %#x", self.global_offset)
# This is a 64 bit archive, find the Zip64EndCD.
else:
locator_real_offset = ecd_real_offset - Zip64CDLocator.sizeof()
backing_store.Seek(locator_real_offset, 0)
locator = Zip64CDLocator(
backing_store.Read(Zip64CDLocator.sizeof()))
if not locator.IsValid():
raise IOError("Zip64CDLocator invalid or not supported.")
# Although it may appear that we can use the Zip64CDLocator to
# locate the Zip64EndCD record via it's offset_of_cd record this
# is not quite so. If the zip file was appended to another file,
# the offset_of_cd field will not be valid, as it still points
# to the old offset. In this case we also need to know the
# global shift.
backing_store.Seek(locator_real_offset - Zip64EndCD.sizeof(),
0)
end_cd = Zip64EndCD(backing_store.Read(Zip64EndCD.sizeof()))
if not end_cd.IsValid():
LOGGER.error("Zip64EndCD magic not correct @%#x",
locator_real_offset - Zip64EndCD.sizeof())
raise RuntimeError("Zip64EndCD magic not correct")
directory_offset = end_cd.offset_of_cd
directory_number_of_entries = end_cd.number_of_entries_in_volume
# The global offset is now known:
self.global_offset = (
# Real offset of the central directory.
locator_real_offset - Zip64EndCD.sizeof() -
end_cd.size_of_cd -
# The directory offset in zip file offsets.
directory_offset)
LOGGER.info("Global offset: %#x", self.global_offset)
# Now iterate over the directory and read all the ZipInfo structs.
entry_offset = directory_offset
for _ in range(directory_number_of_entries):
backing_store.Seek(entry_offset + self.global_offset, 0)
entry = CDFileHeader(backing_store.Read(CDFileHeader.sizeof()))
if not entry.IsValid():
LOGGER.info("CDFileHeader at offset %#x invalid",
entry_offset)
raise RuntimeError()
zip_info = ZipInfo(
filename=backing_store.Read(entry.file_name_length),
local_header_offset=entry.relative_offset_local_header,
compression_method=entry.compression_method,
compress_size=entry.compress_size,
file_size=entry.file_size,
crc32=entry.crc32,
lastmoddate=entry.dosdate,
lastmodtime=entry.dostime)
# Zip64 local header - parse the Zip64 extended information extra field.
# This field isnt a struct, its a serialization
#if zip_info.local_header_offset < 0 or zip_info.local_header_offset == 0xffffffff:
if entry.extra_field_len > 0:
extrabuf = backing_store.Read(entry.extra_field_len)
extra, readbytes = Zip64FileHeaderExtensibleField.FromBuffer(
entry, extrabuf)
extrabuf = extrabuf[readbytes:]
if extra.header_id == 1:
if extra.Get(
"relative_offset_local_header") is not None:
zip_info.local_header_offset = (
extra.Get("relative_offset_local_header"))
if extra.Get("file_size") is not None:
zip_info.file_size = extra.Get("file_size")
if extra.Get("compress_size") is not None:
zip_info.compress_size = extra.Get("compress_size")
#break
LOGGER.info("Found file %s @ %#x", zip_info.filename,
zip_info.local_header_offset)
# Store this information in the resolver. Ths allows
# segments to be directly opened by URN.
member_urn = aff4_utils.urn_from_member_name(
zip_info.filename, self.urn)
self.resolver.Set(member_urn, lexicon.AFF4_TYPE,
rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE))
self.resolver.Set(member_urn, lexicon.AFF4_STORED, self.urn)
self.resolver.Set(member_urn, lexicon.AFF4_STREAM_SIZE,
rdfvalue.XSDInteger(zip_info.file_size))
self.members[member_urn] = zip_info
# Go to the next entry.
entry_offset += (entry.sizeof() + entry.file_name_length +
entry.extra_field_len +
entry.file_comment_length)
def parse_cd(self, backing_store_urn):
with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
# Find the End of Central Directory Record - We read about 4k of
# data and scan for the header from the end, just in case there is
# an archive comment appended to the end.
backing_store.Seek(-BUFF_SIZE, 2)
ecd_real_offset = backing_store.Tell()
buffer = backing_store.Read(BUFF_SIZE)
end_cd, buffer_offset = EndCentralDirectory.FromBuffer(buffer)
ecd_real_offset += buffer_offset
LOGGER.info("Found ECD at %#x", ecd_real_offset)
# Fetch the volume comment.
if end_cd.comment_len > 0:
backing_store.Seek(ecd_real_offset + end_cd.sizeof())
urn_string = backing_store.Read(end_cd.comment_len)
LOGGER.info("Loaded AFF4 volume URN %s from zip file.",
urn_string)
# There is a catch 22 here - before we parse the ZipFile we dont
# know the Volume's URN, but we need to know the URN so the
# AFF4FactoryOpen() can open it. Therefore we start with a random
# URN and then create a new ZipFile volume. After parsing the
# central directory we discover our URN and therefore we can delete
# the old, randomly selected URN.
if self.urn != urn_string:
self.resolver.DeleteSubject(self.urn)
self.urn.Set(urn_string)
# Set these triples so we know how to open the zip file again.
self.resolver.Set(self.urn, lexicon.AFF4_TYPE, rdfvalue.URN(
lexicon.AFF4_ZIP_TYPE))
self.resolver.Set(self.urn, lexicon.AFF4_STORED, rdfvalue.URN(
backing_store_urn))
self.resolver.Set(backing_store_urn, lexicon.AFF4_CONTAINS,
self.urn)
directory_offset = end_cd.offset_of_cd
directory_number_of_entries = end_cd.total_entries_in_cd
# Traditional zip file - non 64 bit.
if directory_offset > 0:
# The global difference between the zip file offsets and real
# file offsets. This is non zero when the zip file was appended
# to another file.
self.global_offset = (
# Real ECD offset.
ecd_real_offset - end_cd.size_of_cd -
# Claimed CD offset.
directory_offset)
LOGGER.info("Global offset: %#x", self.global_offset)
# This is a 64 bit archive, find the Zip64EndCD.
else:
locator_real_offset = ecd_real_offset - Zip64CDLocator.sizeof()
backing_store.Seek(locator_real_offset, 0)
locator = Zip64CDLocator(
backing_store.Read(Zip64CDLocator.sizeof()))
if not locator.IsValid():
raise IOError("Zip64CDLocator invalid or not supported.")
# Although it may appear that we can use the Zip64CDLocator to
# locate the Zip64EndCD record via it's offset_of_cd record this
# is not quite so. If the zip file was appended to another file,
# the offset_of_cd field will not be valid, as it still points
# to the old offset. In this case we also need to know the
# global shift.
backing_store.Seek(
locator_real_offset - Zip64EndCD.sizeof(), 0)
end_cd = Zip64EndCD(
backing_store.Read(Zip64EndCD.sizeof()))
if not end_cd.IsValid():
LOGGER.error("Zip64EndCD magic not correct @%#x",
locator_real_offset - Zip64EndCD.sizeof())
raise RuntimeError("Zip64EndCD magic not correct")
directory_offset = end_cd.offset_of_cd
directory_number_of_entries = end_cd.number_of_entries_in_volume
# The global offset is now known:
self.global_offset = (
# Real offset of the central directory.
locator_real_offset - Zip64EndCD.sizeof() -
end_cd.size_of_cd -
# The directory offset in zip file offsets.
directory_offset)
LOGGER.info("Global offset: %#x", self.global_offset)
# Now iterate over the directory and read all the ZipInfo structs.
entry_offset = directory_offset
for _ in xrange(directory_number_of_entries):
backing_store.Seek(entry_offset + self.global_offset, 0)
entry = CDFileHeader(
backing_store.Read(CDFileHeader.sizeof()))
if not entry.IsValid():
LOGGER.info(
"CDFileHeader at offset %#x invalid", entry_offset)
raise RuntimeError()
zip_info = ZipInfo(
filename=backing_store.Read(entry.file_name_length),
local_header_offset=entry.relative_offset_local_header,
compression_method=entry.compression_method,
compress_size=entry.compress_size,
file_size=entry.file_size,
crc32=entry.crc32,
lastmoddate=entry.dosdate,
lastmodtime=entry.dostime)
# Zip64 local header - parse the extra field.
if zip_info.local_header_offset < 0:
# Parse all the extra field records.
real_end_of_extra = (
backing_store.Tell() + entry.extra_field_len)
while backing_store.Tell() < real_end_of_extra:
extra = Zip64FileHeaderExtensibleField(
backing_store.Read(entry.extra_field_len))
if extra.header_id == 1:
zip_info.local_header_offset = (
extra.relative_offset_local_header)
zip_info.file_size = extra.file_size
zip_info.compress_size = extra.compress_size
break
if zip_info.local_header_offset >= 0:
LOGGER.info("Found file %s @ %#x", zip_info.filename,
zip_info.local_header_offset)
# Store this information in the resolver. Ths allows
# segments to be directly opened by URN.
member_urn = aff4_utils.urn_from_member_name(
zip_info.filename, self.urn)
self.resolver.Set(
member_urn, lexicon.AFF4_TYPE, rdfvalue.URN(
lexicon.AFF4_ZIP_SEGMENT_TYPE))
self.resolver.Set(member_urn, lexicon.AFF4_STORED, self.urn)
self.members[zip_info.filename] = zip_info
# Go to the next entry.
entry_offset += (entry.sizeof() +
entry.file_name_length +
entry.extra_field_len +
entry.file_comment_length)