def CreateMember(self, child_urn):
# Check that child is a relative path in our URN.
relative_path = self.urn.RelativePath(child_urn)
if relative_path == child_urn.SerializeToString():
raise IOError("Child URN is not within container URN.")
# Use this filename. Note that since filesystems can not typically
# represent files and directories as the same path component we can not
# allow slashes in the filename. Otherwise we will fail to create
# e.g. stream/0000000 and stream/0000000/index.
filename = aff4_utils.member_name_for_urn(child_urn,
self.urn,
slash_ok=False)
# We are allowed to create any files inside the directory volume.
self.resolver.Set(child_urn, lexicon.AFF4_TYPE,
rdfvalue.URN(lexicon.AFF4_FILE_TYPE))
self.resolver.Set(child_urn, lexicon.AFF4_STREAM_WRITE_MODE,
rdfvalue.XSDString("truncate"))
self.resolver.Set(child_urn, lexicon.AFF4_DIRECTORY_CHILD_FILENAME,
rdfvalue.XSDString(filename))
# Store the member inside our storage location.
self.resolver.Set(
child_urn, lexicon.AFF4_FILE_NAME,
rdfvalue.XSDString(self.root_path + os.sep + filename))
result = self.resolver.AFF4FactoryOpen(child_urn)
self.MarkDirty()
self.children.add(child_urn)
return result
def CreateMember(self, child_urn):
# Check that child is a relative path in our URN.
relative_path = self.urn.RelativePath(child_urn)
if relative_path == child_urn.SerializeToString():
raise IOError("Child URN is not within container URN.")
# Use this filename. Note that since filesystems can not typically
# represent files and directories as the same path component we can not
# allow slashes in the filename. Otherwise we will fail to create
# e.g. stream/0000000 and stream/0000000/index.
filename = aff4_utils.member_name_for_urn(
child_urn, self.urn, slash_ok=False)
# We are allowed to create any files inside the directory volume.
self.resolver.Set(child_urn, lexicon.AFF4_TYPE,
rdfvalue.URN(lexicon.AFF4_FILE_TYPE))
self.resolver.Set(child_urn, lexicon.AFF4_STREAM_WRITE_MODE,
rdfvalue.XSDString("truncate"))
self.resolver.Set(child_urn, lexicon.AFF4_DIRECTORY_CHILD_FILENAME,
rdfvalue.XSDString(filename))
# Store the member inside our storage location.
self.resolver.Set(
child_urn, lexicon.AFF4_FILE_NAME,
rdfvalue.XSDString(self.root_path + os.sep + filename))
result = self.resolver.AFF4FactoryOpen(child_urn)
self.MarkDirty()
self.children.add(child_urn)
return result
def LoadFromZipFile(self, owner):
"""Read the segment data from the ZipFile owner."""
member_name = aff4_utils.member_name_for_urn(self.urn, owner.urn)
# Parse the ZipFileHeader for this filename.
zip_info = owner.members.get(member_name)
if zip_info is None:
# The owner does not have this file yet - we add it when closing.
self.fd = StringIO.StringIO()
return
backing_store_urn = owner.backing_store_urn
with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
backing_store.Seek(
zip_info.local_header_offset + owner.global_offset, 0)
file_header = ZipFileHeader(
backing_store.Read(ZipFileHeader.sizeof()))
if not file_header.IsValid():
raise IOError("Local file header invalid!")
# The filename should be null terminated.
file_header_filename = backing_store.Read(
file_header.file_name_length).split("\x00")[0]
if file_header_filename != zip_info.filename:
msg = (u"Local filename %s different from "
u"central directory %s.") % (file_header_filename,
zip_info.filename)
LOGGER.error(msg)
raise IOError(msg)
backing_store.Seek(file_header.extra_field_len, aff4.SEEK_CUR)
buffer_size = zip_info.file_size
if file_header.compression_method == ZIP_DEFLATE:
# We write the entire file in a memory buffer if we need to
# deflate it.
self.compression_method = ZIP_DEFLATE
c_buffer = backing_store.Read(zip_info.compress_size)
decomp_buffer = DecompressBuffer(c_buffer)
if len(decomp_buffer) != buffer_size:
LOGGER.info("Unable to decompress file %s", self.urn)
raise IOError()
self.fd = StringIO.StringIO(decomp_buffer)
elif file_header.compression_method == ZIP_STORED:
# Otherwise we map a slice into it.
self.fd = FileWrapper(self.resolver, backing_store_urn,
backing_store.Tell(), buffer_size)
else:
LOGGER.info("Unsupported compression method.")
raise NotImplementedError()
def LoadFromZipFile(self, owner):
"""Read the segment data from the ZipFile owner."""
member_name = aff4_utils.member_name_for_urn(self.urn, owner.urn)
# Parse the ZipFileHeader for this filename.
zip_info = owner.members.get(member_name)
if zip_info is None:
# The owner does not have this file yet - we add it when closing.
self.fd = StringIO.StringIO()
return
backing_store_urn = owner.backing_store_urn
with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
backing_store.Seek(
zip_info.local_header_offset + owner.global_offset, 0)
file_header = ZipFileHeader(
backing_store.Read(ZipFileHeader.sizeof()))
if not file_header.IsValid():
raise IOError("Local file header invalid!")
# The filename should be null terminated.
file_header_filename = backing_store.Read(
file_header.file_name_length).split("\x00")[0]
if file_header_filename != zip_info.filename:
msg = (u"Local filename %s different from "
u"central directory %s.") % (
file_header_filename, zip_info.filename)
LOGGER.error(msg)
raise IOError(msg)
backing_store.Seek(file_header.extra_field_len, aff4.SEEK_CUR)
buffer_size = zip_info.file_size
if file_header.compression_method == ZIP_DEFLATE:
# We write the entire file in a memory buffer if we need to
# deflate it.
self.compression_method = ZIP_DEFLATE
c_buffer = backing_store.Read(zip_info.compress_size)
decomp_buffer = DecompressBuffer(c_buffer)
if len(decomp_buffer) != buffer_size:
LOGGER.info("Unable to decompress file %s", self.urn)
raise IOError()
self.fd = StringIO.StringIO(decomp_buffer)
elif file_header.compression_method == ZIP_STORED:
# Otherwise we map a slice into it.
self.fd = FileWrapper(self.resolver, backing_store_urn,
backing_store.Tell(), buffer_size)
else:
LOGGER.info("Unsupported compression method.")
raise NotImplementedError()
def LoadFromURN(self):
mode = self.resolver.Get(self.urn, lexicon.AFF4_STREAM_WRITE_MODE)
if mode == "append":
raise RuntimeError("Cloud storage does not support appending.")
# If there is a dedicated cache directory we should use it first.
cache_directory = self.resolver.Get(lexicon.AFF4_CONFIG_CACHE_DIR,
lexicon.AFF4_FILE_NAME)
# A persistent cache directory is set.
if cache_directory:
filename = aff4_utils.member_name_for_urn(
self.urn, base_urn=rdfvalue.URN(""),
slash_ok=False)
filename = os.path.join(
unicode(cache_directory), filename)
# When truncating a stream we just overwrite it with new data.
if mode == "truncate":
self.fd = open(filename, "w+b")
else:
try:
self.fd = open(filename, "rb")
self.fd.seek(0, 2)
self.size = self.fd.tell()
except IOError:
# Try to fetch the data so we can write it into the local
# cache file.
LOGGER.info("Creating cached file %s", filename)
aff4_utils.EnsureDirectoryExists(filename)
try:
blob_data = self._fetch_blob_data()
self.fd = open(filename, "w+b")
self.fd.write(blob_data)
except IOError as e:
LOGGER.error(
"Unable to write on cache directory %s: %s",
filename, e)
raise
return
else:
# No dedicated cache directory, so we just get a temp file to work
# from.
blob_data = self._fetch_blob_data()
self.fd = tempfile.NamedTemporaryFile(prefix="pyaff4")
if mode != "truncate":
self.fd.write(blob_data)
def LoadFromURN(self):
mode = self.resolver.Get(self.urn, lexicon.AFF4_STREAM_WRITE_MODE)
if mode == "append":
raise RuntimeError("Cloud storage does not support appending.")
# If there is a dedicated cache directory we should use it first.
cache_directory = self.resolver.Get(lexicon.AFF4_CONFIG_CACHE_DIR,
lexicon.AFF4_FILE_NAME)
# A persistent cache directory is set.
if cache_directory:
filename = aff4_utils.member_name_for_urn(
self.urn, base_urn=rdfvalue.URN(""), slash_ok=False)
filename = os.path.join(str(cache_directory), filename)
# When truncating a stream we just overwrite it with new data.
if mode == "truncate":
self.fd = open(filename, "w+b")
else:
try:
self.fd = open(filename, "rb")
self.fd.seek(0, 2)
self.size = self.fd.tell()
except IOError:
# Try to fetch the data so we can write it into the local
# cache file.
LOGGER.info("Creating cached file %s", filename)
aff4_utils.EnsureDirectoryExists(filename)
try:
blob_data = self._fetch_blob_data()
self.fd = open(filename, "w+b")
self.fd.write(blob_data)
except IOError as e:
LOGGER.error(
"Unable to write on cache directory %s: %s",
filename, e)
raise
return
else:
# No dedicated cache directory, so we just get a temp file to work
# from.
blob_data = self._fetch_blob_data()
self.fd = tempfile.NamedTemporaryFile(prefix="pyaff4")
if mode != "truncate":
self.fd.write(blob_data)
def StreamAddMember(self,
member_urn,
stream,
compression_method=ZIP_STORED,
progress=None):
"""An efficient interface to add a new archive member.
Args:
member_urn: The new member URN to be added.
stream: A file-like object (with read() method) that generates data to
be written as the member.
compression_method: How to compress the member.
"""
if progress is None:
progress = aff4.EMPTY_PROGRESS
backing_store_urn = self.resolver.Get(self.urn, lexicon.AFF4_STORED)
with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
LOGGER.info("Writing member %s", member_urn)
# Append member at the end of the file.
backing_store.Seek(0, aff4.SEEK_END)
# zip_info offsets are relative to the start of the zip file (take
# global_offset into account).
zip_info = ZipInfo(
local_header_offset=backing_store.Tell() - self.global_offset,
filename=aff4_utils.member_name_for_urn(member_urn, self.urn),
file_size=0,
crc32=0,
compression_method=compression_method)
# For now we do not support streamed writing so we need to seek back
# to this position later with an updated crc32.
zip_info.WriteFileHeader(backing_store)
if compression_method == ZIP_DEFLATE:
zip_info.compression_method = ZIP_DEFLATE
compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
while True:
data = stream.read(BUFF_SIZE)
if not data:
break
c_data = compressor.compress(data)
zip_info.compress_size += len(c_data)
zip_info.file_size += len(data)
# Python 2 erronously returns a signed int here.
zip_info.crc32 = zlib.crc32(data,
zip_info.crc32) & 0xffffffff
backing_store.Write(c_data)
progress.Report(zip_info.file_size)
# Finalize the compressor.
c_data = compressor.flush()
zip_info.compress_size += len(c_data)
backing_store.Write(c_data)
# Just write the data directly.
elif compression_method == ZIP_STORED:
zip_info.compression_method = ZIP_STORED
while True:
data = stream.read(BUFF_SIZE)
if not data:
break
zip_info.compress_size += len(data)
zip_info.file_size += len(data)
# Python 2 erronously returns a signed int here.
zip_info.crc32 = zlib.crc32(data,
zip_info.crc32) & 0xffffffff
progress.Report(zip_info.file_size)
backing_store.Write(data)
else:
raise RuntimeError("Unsupported compression method")
# Update the local file header now that CRC32 is calculated.
zip_info.WriteFileHeader(backing_store)
self.members[member_urn] = zip_info
def CreateMember(self, child_urn):
member_filename = aff4_utils.member_name_for_urn(child_urn, self.urn)
return self.CreateZipSegment(member_filename)
def StreamAddMember(self, member_urn, stream,
compression_method=ZIP_STORED,
progress=None):
"""An efficient interface to add a new archive member.
Args:
member_urn: The new member URN to be added.
stream: A file-like object (with read() method) that generates data to
be written as the member.
compression_method: How to compress the member.
"""
if progress is None:
progress = aff4.EMPTY_PROGRESS
backing_store_urn = self.resolver.Get(self.urn, lexicon.AFF4_STORED)
with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
LOGGER.info("Writing member %s", member_urn)
# Append member at the end of the file.
backing_store.Seek(0, aff4.SEEK_END)
# zip_info offsets are relative to the start of the zip file (take
# global_offset into account).
zip_info = ZipInfo(
local_header_offset=backing_store.Tell() - self.global_offset,
filename=aff4_utils.member_name_for_urn(member_urn, self.urn),
file_size=0, crc32=0)
# For now we do not support streamed writing so we need to seek back
# to this position later with an updated crc32.
zip_info.WriteFileHeader(backing_store)
if compression_method == ZIP_DEFLATE:
zip_info.compression_method = ZIP_DEFLATE
compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
while True:
data = stream.read(BUFF_SIZE)
if not data:
break
c_data = compressor.compress(data)
zip_info.compress_size += len(c_data)
zip_info.file_size += len(data)
zip_info.crc32 = zlib.crc32(data, zip_info.crc32)
backing_store.Write(c_data)
progress.Report(zip_info.file_size)
# Finalize the compressor.
c_data = compressor.flush()
zip_info.compress_size += len(c_data)
backing_store.Write(c_data)
# Just write the data directly.
elif compression_method == ZIP_STORED:
zip_info.compression_method = ZIP_STORED
while True:
data = stream.read(BUFF_SIZE)
if not data:
break
zip_info.compress_size += len(data)
zip_info.file_size += len(data)
zip_info.crc32 = zlib.crc32(data, zip_info.crc32)
progress.Report(zip_info.file_size)
backing_store.Write(data)
else:
raise RuntimeError("Unsupported compression method")
# Update the local file header now that CRC32 is calculated.
zip_info.WriteFileHeader(backing_store)
self.members[member_urn] = zip_info
def CreateMember(self, child_urn):
member_filename = aff4_utils.member_name_for_urn(child_urn, self.urn)
return self.CreateZipSegment(member_filename)
