def CreateZipSegment(self, filename): self.MarkDirty() segment_urn = aff4_utils.urn_from_member_name(filename, self.urn) # Is it in the cache? res = self.resolver.CacheGet(segment_urn) if res: return res self.resolver.Set(segment_urn, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE)) self.resolver.Set(segment_urn, lexicon.AFF4_STORED, self.urn) # Keep track of all the segments we issue. self.children.add(segment_urn) result = ZipFileSegment(resolver=self.resolver, urn=segment_urn) result.LoadFromZipFile(self) LOGGER.info("Creating ZipFileSegment %s", result.urn.SerializeToString()) # Add the new object to the object cache. return self.resolver.CachePut(result)
def CreateZipSegment(self, filename): self.MarkDirty() segment_urn = aff4_utils.urn_from_member_name(filename, self.urn) # Is it in the cache? res = self.resolver.CacheGet(segment_urn) if res: return res self.resolver.Set( segment_urn, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE)) self.resolver.Set(segment_urn, lexicon.AFF4_STORED, self.urn) # Keep track of all the segments we issue. self.children.add(segment_urn) result = ZipFileSegment(resolver=self.resolver, urn=segment_urn) result.LoadFromZipFile(self) LOGGER.info("Creating ZipFileSegment %s", result.urn.SerializeToString()) # Add the new object to the object cache. return self.resolver.CachePut(result)
def OpenZipSegment(self, filename): # Is it already in the cache? segment_urn = aff4_utils.urn_from_member_name(filename, self.urn) if segment_urn not in self.members: raise IOError("Segment %s does not exist yet" % filename) res = self.resolver.CacheGet(segment_urn) if res: LOGGER.info("Openning ZipFileSegment (cached) %s", res.urn) return res result = ZipFileSegment(resolver=self.resolver, urn=segment_urn) result.LoadFromZipFile(owner=self) LOGGER.info("Openning ZipFileSegment %s", result.urn) return self.resolver.CachePut(result)
def OpenZipSegment(self, filename): if filename not in self.members: raise IOError("Segment %s does not exist yet" % filename) # Is it already in the cache? segment_urn = aff4_utils.urn_from_member_name(filename, self.urn) res = self.resolver.CacheGet(segment_urn) if res: LOGGER.info("Openning ZipFileSegment (cached) %s", res.urn) return res result = ZipFileSegment(resolver=self.resolver, urn=segment_urn) result.LoadFromZipFile(owner=self) LOGGER.info("Openning ZipFileSegment %s", result.urn) return self.resolver.CachePut(result)
def parse_cd(self, backing_store_urn): with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: # Find the End of Central Directory Record - We read about 4k of # data and scan for the header from the end, just in case there is # an archive comment appended to the end. backing_store.Seek(-BUFF_SIZE, 2) ecd_real_offset = backing_store.Tell() buffer = backing_store.Read(BUFF_SIZE) end_cd, buffer_offset = EndCentralDirectory.FromBuffer(buffer) urn_string = None ecd_real_offset += buffer_offset # Fetch the volume comment. if end_cd.comment_len > 0: backing_store.Seek(ecd_real_offset + end_cd.sizeof()) urn_string = backing_store.Read(end_cd.comment_len) LOGGER.info("Loaded AFF4 volume URN %s from zip file.", urn_string) #if end_cd.size_of_cd == 0xFFFFFFFF: # end_cd, buffer_offset = Zip64EndCD.FromBuffer(buffer) #LOGGER.info("Found ECD at %#x", ecd_real_offset) # There is a catch 22 here - before we parse the ZipFile we dont # know the Volume's URN, but we need to know the URN so the # AFF4FactoryOpen() can open it. Therefore we start with a random # URN and then create a new ZipFile volume. After parsing the # central directory we discover our URN and therefore we can delete # the old, randomly selected URN. if urn_string and self.urn != urn_string: self.resolver.DeleteSubject(self.urn) self.urn.Set(utils.SmartUnicode(urn_string)) # Set these triples so we know how to open the zip file again. self.resolver.Set(self.urn, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_ZIP_TYPE)) self.resolver.Set(self.urn, lexicon.AFF4_STORED, rdfvalue.URN(backing_store_urn)) self.resolver.Set(backing_store_urn, lexicon.AFF4_CONTAINS, self.urn) directory_offset = end_cd.offset_of_cd directory_number_of_entries = end_cd.total_entries_in_cd # Traditional zip file - non 64 bit. if directory_offset > 0 and directory_offset != 0xffffffff: # The global difference between the zip file offsets and real # file offsets. This is non zero when the zip file was appended # to another file. self.global_offset = ( # Real ECD offset. ecd_real_offset - end_cd.size_of_cd - # Claimed CD offset. directory_offset) LOGGER.info("Global offset: %#x", self.global_offset) # This is a 64 bit archive, find the Zip64EndCD. else: locator_real_offset = ecd_real_offset - Zip64CDLocator.sizeof() backing_store.Seek(locator_real_offset, 0) locator = Zip64CDLocator( backing_store.Read(Zip64CDLocator.sizeof())) if not locator.IsValid(): raise IOError("Zip64CDLocator invalid or not supported.") # Although it may appear that we can use the Zip64CDLocator to # locate the Zip64EndCD record via it's offset_of_cd record this # is not quite so. If the zip file was appended to another file, # the offset_of_cd field will not be valid, as it still points # to the old offset. In this case we also need to know the # global shift. backing_store.Seek(locator_real_offset - Zip64EndCD.sizeof(), 0) end_cd = Zip64EndCD(backing_store.Read(Zip64EndCD.sizeof())) if not end_cd.IsValid(): LOGGER.error("Zip64EndCD magic not correct @%#x", locator_real_offset - Zip64EndCD.sizeof()) raise RuntimeError("Zip64EndCD magic not correct") directory_offset = end_cd.offset_of_cd directory_number_of_entries = end_cd.number_of_entries_in_volume # The global offset is now known: self.global_offset = ( # Real offset of the central directory. locator_real_offset - Zip64EndCD.sizeof() - end_cd.size_of_cd - # The directory offset in zip file offsets. directory_offset) LOGGER.info("Global offset: %#x", self.global_offset) # Now iterate over the directory and read all the ZipInfo structs. entry_offset = directory_offset for _ in range(directory_number_of_entries): backing_store.Seek(entry_offset + self.global_offset, 0) entry = CDFileHeader(backing_store.Read(CDFileHeader.sizeof())) if not entry.IsValid(): LOGGER.info("CDFileHeader at offset %#x invalid", entry_offset) raise RuntimeError() zip_info = ZipInfo( filename=backing_store.Read(entry.file_name_length), local_header_offset=entry.relative_offset_local_header, compression_method=entry.compression_method, compress_size=entry.compress_size, file_size=entry.file_size, crc32=entry.crc32, lastmoddate=entry.dosdate, lastmodtime=entry.dostime) # Zip64 local header - parse the Zip64 extended information extra field. # This field isnt a struct, its a serialization #if zip_info.local_header_offset < 0 or zip_info.local_header_offset == 0xffffffff: if entry.extra_field_len > 0: extrabuf = backing_store.Read(entry.extra_field_len) extra, readbytes = Zip64FileHeaderExtensibleField.FromBuffer( entry, extrabuf) extrabuf = extrabuf[readbytes:] if extra.header_id == 1: if extra.Get( "relative_offset_local_header") is not None: zip_info.local_header_offset = ( extra.Get("relative_offset_local_header")) if extra.Get("file_size") is not None: zip_info.file_size = extra.Get("file_size") if extra.Get("compress_size") is not None: zip_info.compress_size = extra.Get("compress_size") #break LOGGER.info("Found file %s @ %#x", zip_info.filename, zip_info.local_header_offset) # Store this information in the resolver. Ths allows # segments to be directly opened by URN. member_urn = aff4_utils.urn_from_member_name( zip_info.filename, self.urn) self.resolver.Set(member_urn, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE)) self.resolver.Set(member_urn, lexicon.AFF4_STORED, self.urn) self.resolver.Set(member_urn, lexicon.AFF4_STREAM_SIZE, rdfvalue.XSDInteger(zip_info.file_size)) self.members[member_urn] = zip_info # Go to the next entry. entry_offset += (entry.sizeof() + entry.file_name_length + entry.extra_field_len + entry.file_comment_length)
def parse_cd(self, backing_store_urn): with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: # Find the End of Central Directory Record - We read about 4k of # data and scan for the header from the end, just in case there is # an archive comment appended to the end. backing_store.Seek(-BUFF_SIZE, 2) ecd_real_offset = backing_store.Tell() buffer = backing_store.Read(BUFF_SIZE) end_cd, buffer_offset = EndCentralDirectory.FromBuffer(buffer) ecd_real_offset += buffer_offset LOGGER.info("Found ECD at %#x", ecd_real_offset) # Fetch the volume comment. if end_cd.comment_len > 0: backing_store.Seek(ecd_real_offset + end_cd.sizeof()) urn_string = backing_store.Read(end_cd.comment_len) LOGGER.info("Loaded AFF4 volume URN %s from zip file.", urn_string) # There is a catch 22 here - before we parse the ZipFile we dont # know the Volume's URN, but we need to know the URN so the # AFF4FactoryOpen() can open it. Therefore we start with a random # URN and then create a new ZipFile volume. After parsing the # central directory we discover our URN and therefore we can delete # the old, randomly selected URN. if self.urn != urn_string: self.resolver.DeleteSubject(self.urn) self.urn.Set(urn_string) # Set these triples so we know how to open the zip file again. self.resolver.Set(self.urn, lexicon.AFF4_TYPE, rdfvalue.URN( lexicon.AFF4_ZIP_TYPE)) self.resolver.Set(self.urn, lexicon.AFF4_STORED, rdfvalue.URN( backing_store_urn)) self.resolver.Set(backing_store_urn, lexicon.AFF4_CONTAINS, self.urn) directory_offset = end_cd.offset_of_cd directory_number_of_entries = end_cd.total_entries_in_cd # Traditional zip file - non 64 bit. if directory_offset > 0: # The global difference between the zip file offsets and real # file offsets. This is non zero when the zip file was appended # to another file. self.global_offset = ( # Real ECD offset. ecd_real_offset - end_cd.size_of_cd - # Claimed CD offset. directory_offset) LOGGER.info("Global offset: %#x", self.global_offset) # This is a 64 bit archive, find the Zip64EndCD. else: locator_real_offset = ecd_real_offset - Zip64CDLocator.sizeof() backing_store.Seek(locator_real_offset, 0) locator = Zip64CDLocator( backing_store.Read(Zip64CDLocator.sizeof())) if not locator.IsValid(): raise IOError("Zip64CDLocator invalid or not supported.") # Although it may appear that we can use the Zip64CDLocator to # locate the Zip64EndCD record via it's offset_of_cd record this # is not quite so. If the zip file was appended to another file, # the offset_of_cd field will not be valid, as it still points # to the old offset. In this case we also need to know the # global shift. backing_store.Seek( locator_real_offset - Zip64EndCD.sizeof(), 0) end_cd = Zip64EndCD( backing_store.Read(Zip64EndCD.sizeof())) if not end_cd.IsValid(): LOGGER.error("Zip64EndCD magic not correct @%#x", locator_real_offset - Zip64EndCD.sizeof()) raise RuntimeError("Zip64EndCD magic not correct") directory_offset = end_cd.offset_of_cd directory_number_of_entries = end_cd.number_of_entries_in_volume # The global offset is now known: self.global_offset = ( # Real offset of the central directory. locator_real_offset - Zip64EndCD.sizeof() - end_cd.size_of_cd - # The directory offset in zip file offsets. directory_offset) LOGGER.info("Global offset: %#x", self.global_offset) # Now iterate over the directory and read all the ZipInfo structs. entry_offset = directory_offset for _ in xrange(directory_number_of_entries): backing_store.Seek(entry_offset + self.global_offset, 0) entry = CDFileHeader( backing_store.Read(CDFileHeader.sizeof())) if not entry.IsValid(): LOGGER.info( "CDFileHeader at offset %#x invalid", entry_offset) raise RuntimeError() zip_info = ZipInfo( filename=backing_store.Read(entry.file_name_length), local_header_offset=entry.relative_offset_local_header, compression_method=entry.compression_method, compress_size=entry.compress_size, file_size=entry.file_size, crc32=entry.crc32, lastmoddate=entry.dosdate, lastmodtime=entry.dostime) # Zip64 local header - parse the extra field. if zip_info.local_header_offset < 0: # Parse all the extra field records. real_end_of_extra = ( backing_store.Tell() + entry.extra_field_len) while backing_store.Tell() < real_end_of_extra: extra = Zip64FileHeaderExtensibleField( backing_store.Read(entry.extra_field_len)) if extra.header_id == 1: zip_info.local_header_offset = ( extra.relative_offset_local_header) zip_info.file_size = extra.file_size zip_info.compress_size = extra.compress_size break if zip_info.local_header_offset >= 0: LOGGER.info("Found file %s @ %#x", zip_info.filename, zip_info.local_header_offset) # Store this information in the resolver. Ths allows # segments to be directly opened by URN. member_urn = aff4_utils.urn_from_member_name( zip_info.filename, self.urn) self.resolver.Set( member_urn, lexicon.AFF4_TYPE, rdfvalue.URN( lexicon.AFF4_ZIP_SEGMENT_TYPE)) self.resolver.Set(member_urn, lexicon.AFF4_STORED, self.urn) self.members[zip_info.filename] = zip_info # Go to the next entry. entry_offset += (entry.sizeof() + entry.file_name_length + entry.extra_field_len + entry.file_comment_length)