def CreateMember(self, child_urn):
        # Check that child is a relative path in our URN.
        relative_path = self.urn.RelativePath(child_urn)
        if relative_path == child_urn.SerializeToString():
            raise IOError("Child URN is not within container URN.")

        # Use this filename. Note that since filesystems can not typically
        # represent files and directories as the same path component we can not
        # allow slashes in the filename. Otherwise we will fail to create
        # e.g. stream/0000000 and stream/0000000/index.
        filename = aff4_utils.member_name_for_urn(child_urn,
                                                  self.urn,
                                                  slash_ok=False)

        # We are allowed to create any files inside the directory volume.
        self.resolver.Set(child_urn, lexicon.AFF4_TYPE,
                          rdfvalue.URN(lexicon.AFF4_FILE_TYPE))
        self.resolver.Set(child_urn, lexicon.AFF4_STREAM_WRITE_MODE,
                          rdfvalue.XSDString("truncate"))
        self.resolver.Set(child_urn, lexicon.AFF4_DIRECTORY_CHILD_FILENAME,
                          rdfvalue.XSDString(filename))

        # Store the member inside our storage location.
        self.resolver.Set(
            child_urn, lexicon.AFF4_FILE_NAME,
            rdfvalue.XSDString(self.root_path + os.sep + filename))

        result = self.resolver.AFF4FactoryOpen(child_urn)
        self.MarkDirty()
        self.children.add(child_urn)

        return result
Exemple #2
0
    def CreateMember(self, child_urn):
        # Check that child is a relative path in our URN.
        relative_path = self.urn.RelativePath(child_urn)
        if relative_path == child_urn.SerializeToString():
            raise IOError("Child URN is not within container URN.")

        # Use this filename. Note that since filesystems can not typically
        # represent files and directories as the same path component we can not
        # allow slashes in the filename. Otherwise we will fail to create
        # e.g. stream/0000000 and stream/0000000/index.
        filename = aff4_utils.member_name_for_urn(
            child_urn, self.urn, slash_ok=False)

        # We are allowed to create any files inside the directory volume.
        self.resolver.Set(child_urn, lexicon.AFF4_TYPE,
                          rdfvalue.URN(lexicon.AFF4_FILE_TYPE))
        self.resolver.Set(child_urn, lexicon.AFF4_STREAM_WRITE_MODE,
                          rdfvalue.XSDString("truncate"))
        self.resolver.Set(child_urn, lexicon.AFF4_DIRECTORY_CHILD_FILENAME,
                          rdfvalue.XSDString(filename))

        # Store the member inside our storage location.
        self.resolver.Set(
            child_urn, lexicon.AFF4_FILE_NAME,
            rdfvalue.XSDString(self.root_path + os.sep + filename))

        result = self.resolver.AFF4FactoryOpen(child_urn)
        self.MarkDirty()
        self.children.add(child_urn)

        return result
Exemple #3
0
    def LoadFromZipFile(self, owner):
        """Read the segment data from the ZipFile owner."""
        member_name = aff4_utils.member_name_for_urn(self.urn, owner.urn)

        # Parse the ZipFileHeader for this filename.
        zip_info = owner.members.get(member_name)
        if zip_info is None:
            # The owner does not have this file yet - we add it when closing.
            self.fd = StringIO.StringIO()
            return

        backing_store_urn = owner.backing_store_urn
        with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
            backing_store.Seek(
                zip_info.local_header_offset + owner.global_offset, 0)
            file_header = ZipFileHeader(
                backing_store.Read(ZipFileHeader.sizeof()))

            if not file_header.IsValid():
                raise IOError("Local file header invalid!")

            # The filename should be null terminated.
            file_header_filename = backing_store.Read(
                file_header.file_name_length).split("\x00")[0]

            if file_header_filename != zip_info.filename:
                msg = (u"Local filename %s different from "
                       u"central directory %s.") % (file_header_filename,
                                                    zip_info.filename)
                LOGGER.error(msg)
                raise IOError(msg)

            backing_store.Seek(file_header.extra_field_len, aff4.SEEK_CUR)

            buffer_size = zip_info.file_size
            if file_header.compression_method == ZIP_DEFLATE:
                # We write the entire file in a memory buffer if we need to
                # deflate it.
                self.compression_method = ZIP_DEFLATE
                c_buffer = backing_store.Read(zip_info.compress_size)
                decomp_buffer = DecompressBuffer(c_buffer)
                if len(decomp_buffer) != buffer_size:
                    LOGGER.info("Unable to decompress file %s", self.urn)
                    raise IOError()

                self.fd = StringIO.StringIO(decomp_buffer)

            elif file_header.compression_method == ZIP_STORED:
                # Otherwise we map a slice into it.
                self.fd = FileWrapper(self.resolver, backing_store_urn,
                                      backing_store.Tell(), buffer_size)

            else:
                LOGGER.info("Unsupported compression method.")
                raise NotImplementedError()
Exemple #4
0
    def LoadFromZipFile(self, owner):
        """Read the segment data from the ZipFile owner."""
        member_name = aff4_utils.member_name_for_urn(self.urn, owner.urn)

        # Parse the ZipFileHeader for this filename.
        zip_info = owner.members.get(member_name)
        if zip_info is None:
            # The owner does not have this file yet - we add it when closing.
            self.fd = StringIO.StringIO()
            return

        backing_store_urn = owner.backing_store_urn
        with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
            backing_store.Seek(
                zip_info.local_header_offset + owner.global_offset, 0)
            file_header = ZipFileHeader(
                backing_store.Read(ZipFileHeader.sizeof()))

            if not file_header.IsValid():
                raise IOError("Local file header invalid!")

            # The filename should be null terminated.
            file_header_filename = backing_store.Read(
                file_header.file_name_length).split("\x00")[0]

            if file_header_filename != zip_info.filename:
                msg = (u"Local filename %s different from "
                       u"central directory %s.") % (
                           file_header_filename, zip_info.filename)
                LOGGER.error(msg)
                raise IOError(msg)

            backing_store.Seek(file_header.extra_field_len, aff4.SEEK_CUR)

            buffer_size = zip_info.file_size
            if file_header.compression_method == ZIP_DEFLATE:
                # We write the entire file in a memory buffer if we need to
                # deflate it.
                self.compression_method = ZIP_DEFLATE
                c_buffer = backing_store.Read(zip_info.compress_size)
                decomp_buffer = DecompressBuffer(c_buffer)
                if len(decomp_buffer) != buffer_size:
                    LOGGER.info("Unable to decompress file %s", self.urn)
                    raise IOError()

                self.fd = StringIO.StringIO(decomp_buffer)

            elif file_header.compression_method == ZIP_STORED:
                # Otherwise we map a slice into it.
                self.fd = FileWrapper(self.resolver, backing_store_urn,
                                      backing_store.Tell(), buffer_size)

            else:
                LOGGER.info("Unsupported compression method.")
                raise NotImplementedError()
Exemple #5
0
    def LoadFromURN(self):
        mode = self.resolver.Get(self.urn, lexicon.AFF4_STREAM_WRITE_MODE)
        if mode == "append":
            raise RuntimeError("Cloud storage does not support appending.")

        # If there is a dedicated cache directory we should use it first.
        cache_directory = self.resolver.Get(lexicon.AFF4_CONFIG_CACHE_DIR,
                                            lexicon.AFF4_FILE_NAME)

        # A persistent cache directory is set.
        if cache_directory:
            filename = aff4_utils.member_name_for_urn(
                self.urn, base_urn=rdfvalue.URN(""),
                slash_ok=False)

            filename = os.path.join(
                unicode(cache_directory), filename)

            # When truncating a stream we just overwrite it with new data.
            if mode == "truncate":
                self.fd = open(filename, "w+b")
            else:
                try:
                    self.fd = open(filename, "rb")
                    self.fd.seek(0, 2)
                    self.size = self.fd.tell()
                except IOError:
                    # Try to fetch the data so we can write it into the local
                    # cache file.
                    LOGGER.info("Creating cached file %s", filename)
                    aff4_utils.EnsureDirectoryExists(filename)
                    try:
                        blob_data = self._fetch_blob_data()
                        self.fd = open(filename, "w+b")
                        self.fd.write(blob_data)
                    except IOError as e:
                        LOGGER.error(
                            "Unable to write on cache directory %s: %s",
                            filename, e)

                        raise

                return

        else:
            # No dedicated cache directory, so we just get a temp file to work
            # from.
            blob_data = self._fetch_blob_data()
            self.fd = tempfile.NamedTemporaryFile(prefix="pyaff4")
            if mode != "truncate":
                self.fd.write(blob_data)
Exemple #6
0
    def LoadFromURN(self):
        mode = self.resolver.Get(self.urn, lexicon.AFF4_STREAM_WRITE_MODE)
        if mode == "append":
            raise RuntimeError("Cloud storage does not support appending.")

        # If there is a dedicated cache directory we should use it first.
        cache_directory = self.resolver.Get(lexicon.AFF4_CONFIG_CACHE_DIR,
                                            lexicon.AFF4_FILE_NAME)

        # A persistent cache directory is set.
        if cache_directory:
            filename = aff4_utils.member_name_for_urn(
                self.urn, base_urn=rdfvalue.URN(""), slash_ok=False)

            filename = os.path.join(str(cache_directory), filename)

            # When truncating a stream we just overwrite it with new data.
            if mode == "truncate":
                self.fd = open(filename, "w+b")
            else:
                try:
                    self.fd = open(filename, "rb")
                    self.fd.seek(0, 2)
                    self.size = self.fd.tell()
                except IOError:
                    # Try to fetch the data so we can write it into the local
                    # cache file.
                    LOGGER.info("Creating cached file %s", filename)
                    aff4_utils.EnsureDirectoryExists(filename)
                    try:
                        blob_data = self._fetch_blob_data()
                        self.fd = open(filename, "w+b")
                        self.fd.write(blob_data)
                    except IOError as e:
                        LOGGER.error(
                            "Unable to write on cache directory %s: %s",
                            filename, e)

                        raise

                return

        else:
            # No dedicated cache directory, so we just get a temp file to work
            # from.
            blob_data = self._fetch_blob_data()
            self.fd = tempfile.NamedTemporaryFile(prefix="pyaff4")
            if mode != "truncate":
                self.fd.write(blob_data)
Exemple #7
0
    def StreamAddMember(self,
                        member_urn,
                        stream,
                        compression_method=ZIP_STORED,
                        progress=None):
        """An efficient interface to add a new archive member.

        Args:
          member_urn: The new member URN to be added.
          stream: A file-like object (with read() method) that generates data to
            be written as the member.
          compression_method: How to compress the member.

        """
        if progress is None:
            progress = aff4.EMPTY_PROGRESS

        backing_store_urn = self.resolver.Get(self.urn, lexicon.AFF4_STORED)
        with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
            LOGGER.info("Writing member %s", member_urn)

            # Append member at the end of the file.
            backing_store.Seek(0, aff4.SEEK_END)

            # zip_info offsets are relative to the start of the zip file (take
            # global_offset into account).
            zip_info = ZipInfo(
                local_header_offset=backing_store.Tell() - self.global_offset,
                filename=aff4_utils.member_name_for_urn(member_urn, self.urn),
                file_size=0,
                crc32=0,
                compression_method=compression_method)

            # For now we do not support streamed writing so we need to seek back
            # to this position later with an updated crc32.
            zip_info.WriteFileHeader(backing_store)
            if compression_method == ZIP_DEFLATE:
                zip_info.compression_method = ZIP_DEFLATE
                compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
                                              zlib.DEFLATED, -15)
                while True:
                    data = stream.read(BUFF_SIZE)
                    if not data:
                        break

                    c_data = compressor.compress(data)
                    zip_info.compress_size += len(c_data)
                    zip_info.file_size += len(data)
                    # Python 2 erronously returns a signed int here.
                    zip_info.crc32 = zlib.crc32(data,
                                                zip_info.crc32) & 0xffffffff
                    backing_store.Write(c_data)
                    progress.Report(zip_info.file_size)

                # Finalize the compressor.
                c_data = compressor.flush()
                zip_info.compress_size += len(c_data)
                backing_store.Write(c_data)

            # Just write the data directly.
            elif compression_method == ZIP_STORED:
                zip_info.compression_method = ZIP_STORED
                while True:
                    data = stream.read(BUFF_SIZE)
                    if not data:
                        break

                    zip_info.compress_size += len(data)
                    zip_info.file_size += len(data)
                    # Python 2 erronously returns a signed int here.
                    zip_info.crc32 = zlib.crc32(data,
                                                zip_info.crc32) & 0xffffffff
                    progress.Report(zip_info.file_size)
                    backing_store.Write(data)
            else:
                raise RuntimeError("Unsupported compression method")

            # Update the local file header now that CRC32 is calculated.
            zip_info.WriteFileHeader(backing_store)
            self.members[member_urn] = zip_info
Exemple #8
0
 def CreateMember(self, child_urn):
     member_filename = aff4_utils.member_name_for_urn(child_urn, self.urn)
     return self.CreateZipSegment(member_filename)
Exemple #9
0
    def StreamAddMember(self, member_urn, stream,
                        compression_method=ZIP_STORED,
                        progress=None):
        """An efficient interface to add a new archive member.

        Args:
          member_urn: The new member URN to be added.
          stream: A file-like object (with read() method) that generates data to
            be written as the member.
          compression_method: How to compress the member.

        """
        if progress is None:
            progress = aff4.EMPTY_PROGRESS

        backing_store_urn = self.resolver.Get(self.urn, lexicon.AFF4_STORED)
        with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
            LOGGER.info("Writing member %s", member_urn)

            # Append member at the end of the file.
            backing_store.Seek(0, aff4.SEEK_END)

            # zip_info offsets are relative to the start of the zip file (take
            # global_offset into account).
            zip_info = ZipInfo(
                local_header_offset=backing_store.Tell() - self.global_offset,
                filename=aff4_utils.member_name_for_urn(member_urn, self.urn),
                file_size=0, crc32=0)

            # For now we do not support streamed writing so we need to seek back
            # to this position later with an updated crc32.
            zip_info.WriteFileHeader(backing_store)

            if compression_method == ZIP_DEFLATE:
                zip_info.compression_method = ZIP_DEFLATE
                compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
                                              zlib.DEFLATED, -15)
                while True:
                    data = stream.read(BUFF_SIZE)
                    if not data:
                        break

                    c_data = compressor.compress(data)
                    zip_info.compress_size += len(c_data)
                    zip_info.file_size += len(data)
                    zip_info.crc32 = zlib.crc32(data, zip_info.crc32)
                    backing_store.Write(c_data)
                    progress.Report(zip_info.file_size)

                # Finalize the compressor.
                c_data = compressor.flush()
                zip_info.compress_size += len(c_data)
                backing_store.Write(c_data)

            # Just write the data directly.
            elif compression_method == ZIP_STORED:
                zip_info.compression_method = ZIP_STORED
                while True:
                    data = stream.read(BUFF_SIZE)
                    if not data:
                        break

                    zip_info.compress_size += len(data)
                    zip_info.file_size += len(data)
                    zip_info.crc32 = zlib.crc32(data, zip_info.crc32)
                    progress.Report(zip_info.file_size)
                    backing_store.Write(data)
            else:
                raise RuntimeError("Unsupported compression method")

            # Update the local file header now that CRC32 is calculated.
            zip_info.WriteFileHeader(backing_store)
            self.members[member_urn] = zip_info
Exemple #10
0
 def CreateMember(self, child_urn):
     member_filename = aff4_utils.member_name_for_urn(child_urn, self.urn)
     return self.CreateZipSegment(member_filename)