def CreateMember(self, child_urn): # Check that child is a relative path in our URN. relative_path = self.urn.RelativePath(child_urn) if relative_path == child_urn.SerializeToString(): raise IOError("Child URN is not within container URN.") # Use this filename. Note that since filesystems can not typically # represent files and directories as the same path component we can not # allow slashes in the filename. Otherwise we will fail to create # e.g. stream/0000000 and stream/0000000/index. filename = aff4_utils.member_name_for_urn(child_urn, self.urn, slash_ok=False) # We are allowed to create any files inside the directory volume. self.resolver.Set(child_urn, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_FILE_TYPE)) self.resolver.Set(child_urn, lexicon.AFF4_STREAM_WRITE_MODE, rdfvalue.XSDString("truncate")) self.resolver.Set(child_urn, lexicon.AFF4_DIRECTORY_CHILD_FILENAME, rdfvalue.XSDString(filename)) # Store the member inside our storage location. self.resolver.Set( child_urn, lexicon.AFF4_FILE_NAME, rdfvalue.XSDString(self.root_path + os.sep + filename)) result = self.resolver.AFF4FactoryOpen(child_urn) self.MarkDirty() self.children.add(child_urn) return result
def CreateMember(self, child_urn): # Check that child is a relative path in our URN. relative_path = self.urn.RelativePath(child_urn) if relative_path == child_urn.SerializeToString(): raise IOError("Child URN is not within container URN.") # Use this filename. Note that since filesystems can not typically # represent files and directories as the same path component we can not # allow slashes in the filename. Otherwise we will fail to create # e.g. stream/0000000 and stream/0000000/index. filename = aff4_utils.member_name_for_urn( child_urn, self.urn, slash_ok=False) # We are allowed to create any files inside the directory volume. self.resolver.Set(child_urn, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_FILE_TYPE)) self.resolver.Set(child_urn, lexicon.AFF4_STREAM_WRITE_MODE, rdfvalue.XSDString("truncate")) self.resolver.Set(child_urn, lexicon.AFF4_DIRECTORY_CHILD_FILENAME, rdfvalue.XSDString(filename)) # Store the member inside our storage location. self.resolver.Set( child_urn, lexicon.AFF4_FILE_NAME, rdfvalue.XSDString(self.root_path + os.sep + filename)) result = self.resolver.AFF4FactoryOpen(child_urn) self.MarkDirty() self.children.add(child_urn) return result
def LoadFromZipFile(self, owner): """Read the segment data from the ZipFile owner.""" member_name = aff4_utils.member_name_for_urn(self.urn, owner.urn) # Parse the ZipFileHeader for this filename. zip_info = owner.members.get(member_name) if zip_info is None: # The owner does not have this file yet - we add it when closing. self.fd = StringIO.StringIO() return backing_store_urn = owner.backing_store_urn with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: backing_store.Seek( zip_info.local_header_offset + owner.global_offset, 0) file_header = ZipFileHeader( backing_store.Read(ZipFileHeader.sizeof())) if not file_header.IsValid(): raise IOError("Local file header invalid!") # The filename should be null terminated. file_header_filename = backing_store.Read( file_header.file_name_length).split("\x00")[0] if file_header_filename != zip_info.filename: msg = (u"Local filename %s different from " u"central directory %s.") % (file_header_filename, zip_info.filename) LOGGER.error(msg) raise IOError(msg) backing_store.Seek(file_header.extra_field_len, aff4.SEEK_CUR) buffer_size = zip_info.file_size if file_header.compression_method == ZIP_DEFLATE: # We write the entire file in a memory buffer if we need to # deflate it. self.compression_method = ZIP_DEFLATE c_buffer = backing_store.Read(zip_info.compress_size) decomp_buffer = DecompressBuffer(c_buffer) if len(decomp_buffer) != buffer_size: LOGGER.info("Unable to decompress file %s", self.urn) raise IOError() self.fd = StringIO.StringIO(decomp_buffer) elif file_header.compression_method == ZIP_STORED: # Otherwise we map a slice into it. self.fd = FileWrapper(self.resolver, backing_store_urn, backing_store.Tell(), buffer_size) else: LOGGER.info("Unsupported compression method.") raise NotImplementedError()
def LoadFromZipFile(self, owner): """Read the segment data from the ZipFile owner.""" member_name = aff4_utils.member_name_for_urn(self.urn, owner.urn) # Parse the ZipFileHeader for this filename. zip_info = owner.members.get(member_name) if zip_info is None: # The owner does not have this file yet - we add it when closing. self.fd = StringIO.StringIO() return backing_store_urn = owner.backing_store_urn with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: backing_store.Seek( zip_info.local_header_offset + owner.global_offset, 0) file_header = ZipFileHeader( backing_store.Read(ZipFileHeader.sizeof())) if not file_header.IsValid(): raise IOError("Local file header invalid!") # The filename should be null terminated. file_header_filename = backing_store.Read( file_header.file_name_length).split("\x00")[0] if file_header_filename != zip_info.filename: msg = (u"Local filename %s different from " u"central directory %s.") % ( file_header_filename, zip_info.filename) LOGGER.error(msg) raise IOError(msg) backing_store.Seek(file_header.extra_field_len, aff4.SEEK_CUR) buffer_size = zip_info.file_size if file_header.compression_method == ZIP_DEFLATE: # We write the entire file in a memory buffer if we need to # deflate it. self.compression_method = ZIP_DEFLATE c_buffer = backing_store.Read(zip_info.compress_size) decomp_buffer = DecompressBuffer(c_buffer) if len(decomp_buffer) != buffer_size: LOGGER.info("Unable to decompress file %s", self.urn) raise IOError() self.fd = StringIO.StringIO(decomp_buffer) elif file_header.compression_method == ZIP_STORED: # Otherwise we map a slice into it. self.fd = FileWrapper(self.resolver, backing_store_urn, backing_store.Tell(), buffer_size) else: LOGGER.info("Unsupported compression method.") raise NotImplementedError()
def LoadFromURN(self): mode = self.resolver.Get(self.urn, lexicon.AFF4_STREAM_WRITE_MODE) if mode == "append": raise RuntimeError("Cloud storage does not support appending.") # If there is a dedicated cache directory we should use it first. cache_directory = self.resolver.Get(lexicon.AFF4_CONFIG_CACHE_DIR, lexicon.AFF4_FILE_NAME) # A persistent cache directory is set. if cache_directory: filename = aff4_utils.member_name_for_urn( self.urn, base_urn=rdfvalue.URN(""), slash_ok=False) filename = os.path.join( unicode(cache_directory), filename) # When truncating a stream we just overwrite it with new data. if mode == "truncate": self.fd = open(filename, "w+b") else: try: self.fd = open(filename, "rb") self.fd.seek(0, 2) self.size = self.fd.tell() except IOError: # Try to fetch the data so we can write it into the local # cache file. LOGGER.info("Creating cached file %s", filename) aff4_utils.EnsureDirectoryExists(filename) try: blob_data = self._fetch_blob_data() self.fd = open(filename, "w+b") self.fd.write(blob_data) except IOError as e: LOGGER.error( "Unable to write on cache directory %s: %s", filename, e) raise return else: # No dedicated cache directory, so we just get a temp file to work # from. blob_data = self._fetch_blob_data() self.fd = tempfile.NamedTemporaryFile(prefix="pyaff4") if mode != "truncate": self.fd.write(blob_data)
def LoadFromURN(self): mode = self.resolver.Get(self.urn, lexicon.AFF4_STREAM_WRITE_MODE) if mode == "append": raise RuntimeError("Cloud storage does not support appending.") # If there is a dedicated cache directory we should use it first. cache_directory = self.resolver.Get(lexicon.AFF4_CONFIG_CACHE_DIR, lexicon.AFF4_FILE_NAME) # A persistent cache directory is set. if cache_directory: filename = aff4_utils.member_name_for_urn( self.urn, base_urn=rdfvalue.URN(""), slash_ok=False) filename = os.path.join(str(cache_directory), filename) # When truncating a stream we just overwrite it with new data. if mode == "truncate": self.fd = open(filename, "w+b") else: try: self.fd = open(filename, "rb") self.fd.seek(0, 2) self.size = self.fd.tell() except IOError: # Try to fetch the data so we can write it into the local # cache file. LOGGER.info("Creating cached file %s", filename) aff4_utils.EnsureDirectoryExists(filename) try: blob_data = self._fetch_blob_data() self.fd = open(filename, "w+b") self.fd.write(blob_data) except IOError as e: LOGGER.error( "Unable to write on cache directory %s: %s", filename, e) raise return else: # No dedicated cache directory, so we just get a temp file to work # from. blob_data = self._fetch_blob_data() self.fd = tempfile.NamedTemporaryFile(prefix="pyaff4") if mode != "truncate": self.fd.write(blob_data)
def StreamAddMember(self, member_urn, stream, compression_method=ZIP_STORED, progress=None): """An efficient interface to add a new archive member. Args: member_urn: The new member URN to be added. stream: A file-like object (with read() method) that generates data to be written as the member. compression_method: How to compress the member. """ if progress is None: progress = aff4.EMPTY_PROGRESS backing_store_urn = self.resolver.Get(self.urn, lexicon.AFF4_STORED) with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: LOGGER.info("Writing member %s", member_urn) # Append member at the end of the file. backing_store.Seek(0, aff4.SEEK_END) # zip_info offsets are relative to the start of the zip file (take # global_offset into account). zip_info = ZipInfo( local_header_offset=backing_store.Tell() - self.global_offset, filename=aff4_utils.member_name_for_urn(member_urn, self.urn), file_size=0, crc32=0, compression_method=compression_method) # For now we do not support streamed writing so we need to seek back # to this position later with an updated crc32. zip_info.WriteFileHeader(backing_store) if compression_method == ZIP_DEFLATE: zip_info.compression_method = ZIP_DEFLATE compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) while True: data = stream.read(BUFF_SIZE) if not data: break c_data = compressor.compress(data) zip_info.compress_size += len(c_data) zip_info.file_size += len(data) # Python 2 erronously returns a signed int here. zip_info.crc32 = zlib.crc32(data, zip_info.crc32) & 0xffffffff backing_store.Write(c_data) progress.Report(zip_info.file_size) # Finalize the compressor. c_data = compressor.flush() zip_info.compress_size += len(c_data) backing_store.Write(c_data) # Just write the data directly. elif compression_method == ZIP_STORED: zip_info.compression_method = ZIP_STORED while True: data = stream.read(BUFF_SIZE) if not data: break zip_info.compress_size += len(data) zip_info.file_size += len(data) # Python 2 erronously returns a signed int here. zip_info.crc32 = zlib.crc32(data, zip_info.crc32) & 0xffffffff progress.Report(zip_info.file_size) backing_store.Write(data) else: raise RuntimeError("Unsupported compression method") # Update the local file header now that CRC32 is calculated. zip_info.WriteFileHeader(backing_store) self.members[member_urn] = zip_info
def CreateMember(self, child_urn): member_filename = aff4_utils.member_name_for_urn(child_urn, self.urn) return self.CreateZipSegment(member_filename)
def StreamAddMember(self, member_urn, stream, compression_method=ZIP_STORED, progress=None): """An efficient interface to add a new archive member. Args: member_urn: The new member URN to be added. stream: A file-like object (with read() method) that generates data to be written as the member. compression_method: How to compress the member. """ if progress is None: progress = aff4.EMPTY_PROGRESS backing_store_urn = self.resolver.Get(self.urn, lexicon.AFF4_STORED) with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: LOGGER.info("Writing member %s", member_urn) # Append member at the end of the file. backing_store.Seek(0, aff4.SEEK_END) # zip_info offsets are relative to the start of the zip file (take # global_offset into account). zip_info = ZipInfo( local_header_offset=backing_store.Tell() - self.global_offset, filename=aff4_utils.member_name_for_urn(member_urn, self.urn), file_size=0, crc32=0) # For now we do not support streamed writing so we need to seek back # to this position later with an updated crc32. zip_info.WriteFileHeader(backing_store) if compression_method == ZIP_DEFLATE: zip_info.compression_method = ZIP_DEFLATE compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) while True: data = stream.read(BUFF_SIZE) if not data: break c_data = compressor.compress(data) zip_info.compress_size += len(c_data) zip_info.file_size += len(data) zip_info.crc32 = zlib.crc32(data, zip_info.crc32) backing_store.Write(c_data) progress.Report(zip_info.file_size) # Finalize the compressor. c_data = compressor.flush() zip_info.compress_size += len(c_data) backing_store.Write(c_data) # Just write the data directly. elif compression_method == ZIP_STORED: zip_info.compression_method = ZIP_STORED while True: data = stream.read(BUFF_SIZE) if not data: break zip_info.compress_size += len(data) zip_info.file_size += len(data) zip_info.crc32 = zlib.crc32(data, zip_info.crc32) progress.Report(zip_info.file_size) backing_store.Write(data) else: raise RuntimeError("Unsupported compression method") # Update the local file header now that CRC32 is calculated. zip_info.WriteFileHeader(backing_store) self.members[member_urn] = zip_info