Ejemplo n.º 1
0
    def CreateZipSegment(self, filename):
        self.MarkDirty()
        segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)

        # Is it in the cache?
        res = self.resolver.CacheGet(segment_urn)
        if res:
            return res

        self.resolver.Set(segment_urn, lexicon.AFF4_TYPE,
                          rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE))

        self.resolver.Set(segment_urn, lexicon.AFF4_STORED, self.urn)

        #  Keep track of all the segments we issue.
        self.children.add(segment_urn)

        result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
        result.LoadFromZipFile(self)

        LOGGER.info("Creating ZipFileSegment %s",
                    result.urn.SerializeToString())

        # Add the new object to the object cache.
        return self.resolver.CachePut(result)
Ejemplo n.º 2
0
    def CreateZipSegment(self, filename):
        self.MarkDirty()

        segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)

        # Is it in the cache?
        res = self.resolver.CacheGet(segment_urn)
        if res:
            return res

        self.resolver.Set(
            segment_urn, lexicon.AFF4_TYPE,
            rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE))

        self.resolver.Set(segment_urn, lexicon.AFF4_STORED, self.urn)

        #  Keep track of all the segments we issue.
        self.children.add(segment_urn)

        result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
        result.LoadFromZipFile(self)

        LOGGER.info("Creating ZipFileSegment %s",
                    result.urn.SerializeToString())

        # Add the new object to the object cache.
        return self.resolver.CachePut(result)
Ejemplo n.º 3
0
    def OpenZipSegment(self, filename):
        # Is it already in the cache?
        segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)
        if segment_urn not in self.members:
            raise IOError("Segment %s does not exist yet" % filename)

        res = self.resolver.CacheGet(segment_urn)

        if res:
            LOGGER.info("Openning ZipFileSegment (cached) %s", res.urn)
            return res

        result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
        result.LoadFromZipFile(owner=self)

        LOGGER.info("Openning ZipFileSegment %s", result.urn)

        return self.resolver.CachePut(result)
Ejemplo n.º 4
0
    def OpenZipSegment(self, filename):
        if filename not in self.members:
            raise IOError("Segment %s does not exist yet" % filename)

        # Is it already in the cache?
        segment_urn = aff4_utils.urn_from_member_name(filename, self.urn)
        res = self.resolver.CacheGet(segment_urn)

        if res:
            LOGGER.info("Openning ZipFileSegment (cached) %s", res.urn)
            return res

        result = ZipFileSegment(resolver=self.resolver, urn=segment_urn)
        result.LoadFromZipFile(owner=self)

        LOGGER.info("Openning ZipFileSegment %s", result.urn)

        return self.resolver.CachePut(result)
Ejemplo n.º 5
0
    def parse_cd(self, backing_store_urn):
        with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
            # Find the End of Central Directory Record - We read about 4k of
            # data and scan for the header from the end, just in case there is
            # an archive comment appended to the end.
            backing_store.Seek(-BUFF_SIZE, 2)

            ecd_real_offset = backing_store.Tell()
            buffer = backing_store.Read(BUFF_SIZE)

            end_cd, buffer_offset = EndCentralDirectory.FromBuffer(buffer)

            urn_string = None

            ecd_real_offset += buffer_offset

            # Fetch the volume comment.
            if end_cd.comment_len > 0:
                backing_store.Seek(ecd_real_offset + end_cd.sizeof())
                urn_string = backing_store.Read(end_cd.comment_len)

                LOGGER.info("Loaded AFF4 volume URN %s from zip file.",
                            urn_string)

            #if end_cd.size_of_cd == 0xFFFFFFFF:
            #    end_cd, buffer_offset = Zip64EndCD.FromBuffer(buffer)

            #LOGGER.info("Found ECD at %#x", ecd_real_offset)

            # There is a catch 22 here - before we parse the ZipFile we dont
            # know the Volume's URN, but we need to know the URN so the
            # AFF4FactoryOpen() can open it. Therefore we start with a random
            # URN and then create a new ZipFile volume. After parsing the
            # central directory we discover our URN and therefore we can delete
            # the old, randomly selected URN.
            if urn_string and self.urn != urn_string:
                self.resolver.DeleteSubject(self.urn)
                self.urn.Set(utils.SmartUnicode(urn_string))

                # Set these triples so we know how to open the zip file again.
                self.resolver.Set(self.urn, lexicon.AFF4_TYPE,
                                  rdfvalue.URN(lexicon.AFF4_ZIP_TYPE))
                self.resolver.Set(self.urn, lexicon.AFF4_STORED,
                                  rdfvalue.URN(backing_store_urn))
                self.resolver.Set(backing_store_urn, lexicon.AFF4_CONTAINS,
                                  self.urn)

            directory_offset = end_cd.offset_of_cd
            directory_number_of_entries = end_cd.total_entries_in_cd

            # Traditional zip file - non 64 bit.
            if directory_offset > 0 and directory_offset != 0xffffffff:
                # The global difference between the zip file offsets and real
                # file offsets. This is non zero when the zip file was appended
                # to another file.
                self.global_offset = (
                    # Real ECD offset.
                    ecd_real_offset - end_cd.size_of_cd -

                    # Claimed CD offset.
                    directory_offset)

                LOGGER.info("Global offset: %#x", self.global_offset)

            # This is a 64 bit archive, find the Zip64EndCD.
            else:
                locator_real_offset = ecd_real_offset - Zip64CDLocator.sizeof()
                backing_store.Seek(locator_real_offset, 0)
                locator = Zip64CDLocator(
                    backing_store.Read(Zip64CDLocator.sizeof()))

                if not locator.IsValid():
                    raise IOError("Zip64CDLocator invalid or not supported.")

                # Although it may appear that we can use the Zip64CDLocator to
                # locate the Zip64EndCD record via it's offset_of_cd record this
                # is not quite so. If the zip file was appended to another file,
                # the offset_of_cd field will not be valid, as it still points
                # to the old offset. In this case we also need to know the
                # global shift.
                backing_store.Seek(locator_real_offset - Zip64EndCD.sizeof(),
                                   0)

                end_cd = Zip64EndCD(backing_store.Read(Zip64EndCD.sizeof()))

                if not end_cd.IsValid():
                    LOGGER.error("Zip64EndCD magic not correct @%#x",
                                 locator_real_offset - Zip64EndCD.sizeof())
                    raise RuntimeError("Zip64EndCD magic not correct")

                directory_offset = end_cd.offset_of_cd
                directory_number_of_entries = end_cd.number_of_entries_in_volume

                # The global offset is now known:
                self.global_offset = (
                    # Real offset of the central directory.
                    locator_real_offset - Zip64EndCD.sizeof() -
                    end_cd.size_of_cd -

                    # The directory offset in zip file offsets.
                    directory_offset)

                LOGGER.info("Global offset: %#x", self.global_offset)

            # Now iterate over the directory and read all the ZipInfo structs.
            entry_offset = directory_offset
            for _ in range(directory_number_of_entries):
                backing_store.Seek(entry_offset + self.global_offset, 0)
                entry = CDFileHeader(backing_store.Read(CDFileHeader.sizeof()))

                if not entry.IsValid():
                    LOGGER.info("CDFileHeader at offset %#x invalid",
                                entry_offset)
                    raise RuntimeError()

                zip_info = ZipInfo(
                    filename=backing_store.Read(entry.file_name_length),
                    local_header_offset=entry.relative_offset_local_header,
                    compression_method=entry.compression_method,
                    compress_size=entry.compress_size,
                    file_size=entry.file_size,
                    crc32=entry.crc32,
                    lastmoddate=entry.dosdate,
                    lastmodtime=entry.dostime)

                # Zip64 local header - parse the Zip64 extended information extra field.
                # This field isnt a struct, its a serialization
                #if zip_info.local_header_offset < 0 or zip_info.local_header_offset == 0xffffffff:
                if entry.extra_field_len > 0:
                    extrabuf = backing_store.Read(entry.extra_field_len)

                    extra, readbytes = Zip64FileHeaderExtensibleField.FromBuffer(
                        entry, extrabuf)
                    extrabuf = extrabuf[readbytes:]

                    if extra.header_id == 1:
                        if extra.Get(
                                "relative_offset_local_header") is not None:
                            zip_info.local_header_offset = (
                                extra.Get("relative_offset_local_header"))
                        if extra.Get("file_size") is not None:
                            zip_info.file_size = extra.Get("file_size")
                        if extra.Get("compress_size") is not None:
                            zip_info.compress_size = extra.Get("compress_size")
                            #break

                LOGGER.info("Found file %s @ %#x", zip_info.filename,
                            zip_info.local_header_offset)

                # Store this information in the resolver. Ths allows
                # segments to be directly opened by URN.
                member_urn = aff4_utils.urn_from_member_name(
                    zip_info.filename, self.urn)

                self.resolver.Set(member_urn, lexicon.AFF4_TYPE,
                                  rdfvalue.URN(lexicon.AFF4_ZIP_SEGMENT_TYPE))

                self.resolver.Set(member_urn, lexicon.AFF4_STORED, self.urn)
                self.resolver.Set(member_urn, lexicon.AFF4_STREAM_SIZE,
                                  rdfvalue.XSDInteger(zip_info.file_size))
                self.members[member_urn] = zip_info

                # Go to the next entry.
                entry_offset += (entry.sizeof() + entry.file_name_length +
                                 entry.extra_field_len +
                                 entry.file_comment_length)
Ejemplo n.º 6
0
    def parse_cd(self, backing_store_urn):
        with self.resolver.AFF4FactoryOpen(backing_store_urn) as backing_store:
            # Find the End of Central Directory Record - We read about 4k of
            # data and scan for the header from the end, just in case there is
            # an archive comment appended to the end.
            backing_store.Seek(-BUFF_SIZE, 2)

            ecd_real_offset = backing_store.Tell()
            buffer = backing_store.Read(BUFF_SIZE)

            end_cd, buffer_offset = EndCentralDirectory.FromBuffer(buffer)
            ecd_real_offset += buffer_offset

            LOGGER.info("Found ECD at %#x", ecd_real_offset)

            # Fetch the volume comment.
            if end_cd.comment_len > 0:
                backing_store.Seek(ecd_real_offset + end_cd.sizeof())
                urn_string = backing_store.Read(end_cd.comment_len)

                LOGGER.info("Loaded AFF4 volume URN %s from zip file.",
                            urn_string)

            # There is a catch 22 here - before we parse the ZipFile we dont
            # know the Volume's URN, but we need to know the URN so the
            # AFF4FactoryOpen() can open it. Therefore we start with a random
            # URN and then create a new ZipFile volume. After parsing the
            # central directory we discover our URN and therefore we can delete
            # the old, randomly selected URN.
            if self.urn != urn_string:
                self.resolver.DeleteSubject(self.urn)
                self.urn.Set(urn_string)

                # Set these triples so we know how to open the zip file again.
                self.resolver.Set(self.urn, lexicon.AFF4_TYPE, rdfvalue.URN(
                    lexicon.AFF4_ZIP_TYPE))
                self.resolver.Set(self.urn, lexicon.AFF4_STORED, rdfvalue.URN(
                    backing_store_urn))
                self.resolver.Set(backing_store_urn, lexicon.AFF4_CONTAINS,
                                  self.urn)

            directory_offset = end_cd.offset_of_cd
            directory_number_of_entries = end_cd.total_entries_in_cd

            # Traditional zip file - non 64 bit.
            if directory_offset > 0:
                # The global difference between the zip file offsets and real
                # file offsets. This is non zero when the zip file was appended
                # to another file.
                self.global_offset = (
                    # Real ECD offset.
                    ecd_real_offset - end_cd.size_of_cd -

                    # Claimed CD offset.
                    directory_offset)

                LOGGER.info("Global offset: %#x", self.global_offset)

            # This is a 64 bit archive, find the Zip64EndCD.
            else:
                locator_real_offset = ecd_real_offset - Zip64CDLocator.sizeof()
                backing_store.Seek(locator_real_offset, 0)
                locator = Zip64CDLocator(
                    backing_store.Read(Zip64CDLocator.sizeof()))

                if not locator.IsValid():
                    raise IOError("Zip64CDLocator invalid or not supported.")

                # Although it may appear that we can use the Zip64CDLocator to
                # locate the Zip64EndCD record via it's offset_of_cd record this
                # is not quite so. If the zip file was appended to another file,
                # the offset_of_cd field will not be valid, as it still points
                # to the old offset. In this case we also need to know the
                # global shift.
                backing_store.Seek(
                    locator_real_offset - Zip64EndCD.sizeof(), 0)

                end_cd = Zip64EndCD(
                    backing_store.Read(Zip64EndCD.sizeof()))

                if not end_cd.IsValid():
                    LOGGER.error("Zip64EndCD magic not correct @%#x",
                                 locator_real_offset - Zip64EndCD.sizeof())
                    raise RuntimeError("Zip64EndCD magic not correct")

                directory_offset = end_cd.offset_of_cd
                directory_number_of_entries = end_cd.number_of_entries_in_volume

                # The global offset is now known:
                self.global_offset = (
                    # Real offset of the central directory.
                    locator_real_offset - Zip64EndCD.sizeof() -
                    end_cd.size_of_cd -

                    # The directory offset in zip file offsets.
                    directory_offset)

                LOGGER.info("Global offset: %#x", self.global_offset)

            # Now iterate over the directory and read all the ZipInfo structs.
            entry_offset = directory_offset
            for _ in xrange(directory_number_of_entries):
                backing_store.Seek(entry_offset + self.global_offset, 0)
                entry = CDFileHeader(
                    backing_store.Read(CDFileHeader.sizeof()))

                if not entry.IsValid():
                    LOGGER.info(
                        "CDFileHeader at offset %#x invalid", entry_offset)
                    raise RuntimeError()

                zip_info = ZipInfo(
                    filename=backing_store.Read(entry.file_name_length),
                    local_header_offset=entry.relative_offset_local_header,
                    compression_method=entry.compression_method,
                    compress_size=entry.compress_size,
                    file_size=entry.file_size,
                    crc32=entry.crc32,
                    lastmoddate=entry.dosdate,
                    lastmodtime=entry.dostime)

                # Zip64 local header - parse the extra field.
                if zip_info.local_header_offset < 0:
                    # Parse all the extra field records.
                    real_end_of_extra = (
                        backing_store.Tell() + entry.extra_field_len)

                    while backing_store.Tell() < real_end_of_extra:
                        extra = Zip64FileHeaderExtensibleField(
                            backing_store.Read(entry.extra_field_len))

                        if extra.header_id == 1:
                            zip_info.local_header_offset = (
                                extra.relative_offset_local_header)
                            zip_info.file_size = extra.file_size
                            zip_info.compress_size = extra.compress_size
                            break

                if zip_info.local_header_offset >= 0:
                    LOGGER.info("Found file %s @ %#x", zip_info.filename,
                                zip_info.local_header_offset)

                    # Store this information in the resolver. Ths allows
                    # segments to be directly opened by URN.
                    member_urn = aff4_utils.urn_from_member_name(
                        zip_info.filename, self.urn)

                    self.resolver.Set(
                        member_urn, lexicon.AFF4_TYPE, rdfvalue.URN(
                            lexicon.AFF4_ZIP_SEGMENT_TYPE))

                    self.resolver.Set(member_urn, lexicon.AFF4_STORED, self.urn)

                    self.members[zip_info.filename] = zip_info

                # Go to the next entry.
                entry_offset += (entry.sizeof() +
                                 entry.file_name_length +
                                 entry.extra_field_len +
                                 entry.file_comment_length)