Exemple #1
0
 def _offset_v2(self, i):
     """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only 
         be returned if the pack is larger than 4 GiB, or 2^32"""
     offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0]
     
     # if the high-bit is set, this indicates that we have to lookup the offset
     # in the 64 bit region of the file. The current offset ( lower 31 bits )
     # are the index into it
     if offset & 0x80000000:
         offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0]
     # END handle 64 bit offset
     
     return offset
Exemple #2
0
    def _offset_v2(self, i):
        """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only 
			be returned if the pack is larger than 4 GiB, or 2^32"""
        offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0]

        # if the high-bit is set, this indicates that we have to lookup the offset
        # in the 64 bit region of the file. The current offset ( lower 31 bits )
        # are the index into it
        if offset & 0x80000000:
            offset = unpack_from(
                ">Q", self._data,
                self._pack_64_offset + (offset & ~0x80000000) * 8)[0]
        # END handle 64 bit offset

        return offset
Exemple #3
0
    def _set_cache_(self, attr):
        if attr == "_packfile_checksum":
            self._packfile_checksum = self._data[-40:-20]
        elif attr == "_packfile_checksum":
            self._packfile_checksum = self._data[-20:]
        elif attr == "_data":
            # Note: We don't lock the file when reading as we cannot be sure
            # that we can actually write to the location - it could be a read-only
            # alternate for instance
            self._data = file_contents_ro_filepath(self._indexpath)
        else:
            # now its time to initialize everything - if we are here, someone wants
            # to access the fanout table or related properties

            # CHECK VERSION
            self._version = (self._data[:4] == '\377tOc' and 2) or 1
            if self._version == 2:
                version_id = unpack_from(">L", self._data, 4)[0]
                assert version_id == self._version, "Unsupported index version: %i" % version_id
            # END assert version

            # SETUP FUNCTIONS
            # setup our functions according to the actual version
            for fname in ('entry', 'offset', 'sha', 'crc'):
                setattr(self, fname,
                        getattr(self, "_%s_v%i" % (fname, self._version)))
            # END for each function to initialize

            # INITIALIZE DATA
            # byte offset is 8 if version is 2, 0 otherwise
            self._initialize()
Exemple #4
0
	def _set_cache_(self, attr):
		if attr == "_packfile_checksum":
			self._packfile_checksum = self._data[-40:-20]
		elif attr == "_packfile_checksum":
			self._packfile_checksum = self._data[-20:]
		elif attr == "_data":
			# Note: We don't lock the file when reading as we cannot be sure
			# that we can actually write to the location - it could be a read-only
			# alternate for instance
			self._data = file_contents_ro_filepath(self._indexpath)
		else:
			# now its time to initialize everything - if we are here, someone wants
			# to access the fanout table or related properties
			
			# CHECK VERSION
			self._version = (self._data[:4] == '\377tOc' and 2) or 1
			if self._version == 2:
				version_id = unpack_from(">L", self._data, 4)[0] 
				assert version_id == self._version, "Unsupported index version: %i" % version_id
			# END assert version
			
			# SETUP FUNCTIONS
			# setup our functions according to the actual version
			for fname in ('entry', 'offset', 'sha', 'crc'):
				setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version)))
			# END for each function to initialize
			
			
			# INITIALIZE DATA
			# byte offset is 8 if version is 2, 0 otherwise
			self._initialize()
Exemple #5
0
 def _read_fanout(self, byte_offset):
     """Generate a fanout table from our data"""
     d = self._cursor.map()
     out = list()
     append = out.append
     for i in range(256):
         append(unpack_from('>L', d, byte_offset + i*4)[0])
     # END for each entry
     return out
Exemple #6
0
 def _read_fanout(self, byte_offset):
     """Generate a fanout table from our data"""
     d = self._cursor.map()
     out = list()
     append = out.append
     for i in range(256):
         append(unpack_from('>L', d, byte_offset + i * 4)[0])
     # END for each entry
     return out
Exemple #7
0
 def _set_cache_(self, attr):
     # we fill the whole cache, whichever attribute gets queried first
     self._cursor = mman.make_cursor(self._packpath).use_region()
     
     # read the header information
     type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0)
         
     # TODO: figure out whether we should better keep the lock, or maybe
     # add a .keep file instead ?
     if type_id != self.pack_signature:
         raise ParseError("Invalid pack signature: %i" % type_id)
Exemple #8
0
    def _set_cache_(self, attr):
        # we fill the whole cache, whichever attribute gets queried first
        self._cursor = mman.make_cursor(self._packpath).use_region()

        # read the header information
        type_id, self._version, self._size = unpack_from(
            ">LLL", self._cursor.map(), 0)

        # TODO: figure out whether we should better keep the lock, or maybe
        # add a .keep file instead ?
        if type_id != self.pack_signature:
            raise ParseError("Invalid pack signature: %i" % type_id)
Exemple #9
0
	def _set_cache_(self, attr):
		if attr == '_data':
			self._data = file_contents_ro_filepath(self._packpath)
			
			# read the header information
			type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0)
			
			# TODO: figure out whether we should better keep the lock, or maybe
			# add a .keep file instead ?
		else: # must be '_size' or '_version'
			# read header info - we do that just with a file stream
			type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12))
Exemple #10
0
    def _set_cache_(self, attr):
        if attr == '_data':
            self._data = file_contents_ro_filepath(self._packpath)

            # read the header information
            type_id, self._version, self._size = unpack_from(
                ">4sLL", self._data, 0)

            # TODO: figure out whether we should better keep the lock, or maybe
            # add a .keep file instead ?
        else:  # must be '_size' or '_version'
            # read header info - we do that just with a file stream
            type_id, self._version, self._size = unpack(
                ">4sLL",
                open(self._packpath).read(12))
Exemple #11
0
    def _set_cache_(self, attr):
        if attr == "_packfile_checksum":
            self._packfile_checksum = self._cursor.map()[-40:-20]
        elif attr == "_packfile_checksum":
            self._packfile_checksum = self._cursor.map()[-20:]
        elif attr == "_cursor":
            # Note: We don't lock the file when reading as we cannot be sure
            # that we can actually write to the location - it could be a read-only
            # alternate for instance
            self._cursor = mman.make_cursor(self._indexpath).use_region()
            # We will assume that the index will always fully fit into memory !
            if mman.window_size() > 0 and self._cursor.file_size(
            ) > mman.window_size():
                raise AssertionError(
                    "The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation"
                    % (self._indexpath, self._cursor.file_size(),
                       mman.window_size()))
            #END assert window size
        else:
            # now its time to initialize everything - if we are here, someone wants
            # to access the fanout table or related properties

            # CHECK VERSION
            mmap = self._cursor.map()
            self._version = (mmap[:4] == self.index_v2_signature and 2) or 1
            if self._version == 2:
                version_id = unpack_from(">L", mmap, 4)[0]
                assert version_id == self._version, "Unsupported index version: %i" % version_id
            # END assert version

            # SETUP FUNCTIONS
            # setup our functions according to the actual version
            for fname in ('entry', 'offset', 'sha', 'crc'):
                setattr(self, fname,
                        getattr(self, "_%s_v%i" % (fname, self._version)))
            # END for each function to initialize

            # INITIALIZE DATA
            # byte offset is 8 if version is 2, 0 otherwise
            self._initialize()
Exemple #12
0
    def _set_cache_(self, attr):
        if attr == "_packfile_checksum":
            self._packfile_checksum = self._cursor.map()[-40:-20]
        elif attr == "_packfile_checksum":
            self._packfile_checksum = self._cursor.map()[-20:]
        elif attr == "_cursor":
            # Note: We don't lock the file when reading as we cannot be sure
            # that we can actually write to the location - it could be a read-only
            # alternate for instance
            self._cursor = mman.make_cursor(self._indexpath).use_region()
            # We will assume that the index will always fully fit into memory !
            if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size():
                raise AssertionError(
                    "The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation"
                    % (self._indexpath, self._cursor.file_size(), mman.window_size())
                )
                # END assert window size
        else:
            # now its time to initialize everything - if we are here, someone wants
            # to access the fanout table or related properties

            # CHECK VERSION
            mmap = self._cursor.map()
            self._version = (mmap[:4] == self.index_v2_signature and 2) or 1
            if self._version == 2:
                version_id = unpack_from(">L", mmap, 4)[0]
                assert version_id == self._version, "Unsupported index version: %i" % version_id
                # END assert version

                # SETUP FUNCTIONS
                # setup our functions according to the actual version
            for fname in ("entry", "offset", "sha", "crc"):
                setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version)))
                # END for each function to initialize

                # INITIALIZE DATA
                # byte offset is 8 if version is 2, 0 otherwise
            self._initialize()
Exemple #13
0
 def _offset_v1(self, i):
     """see ``_offset_v2``"""
     return unpack_from(">L", self._cursor.map(), 1024 + i * 24)[0]
Exemple #14
0
 def _crc_v2(self, i):
     """:return: 4 bytes crc for the object at index i"""
     return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] 
Exemple #15
0
 def _offset_v1(self, i):
     """see ``_offset_v2``"""
     return unpack_from(">L", self._data, 1024 + i * 24)[0]
Exemple #16
0
	def _offset_v1(self, i):
		"""see ``_offset_v2``"""
		return unpack_from(">L", self._data, 1024 + i*24)[0]
Exemple #17
0
 def _offset_v1(self, i):
     """see ``_offset_v2``"""
     return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0]
Exemple #18
0
 def _entry_v1(self, i):
     """:return: tuple(offset, binsha, 0)"""
     return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) 
Exemple #19
0
 def _entry_v1(self, i):
     """:return: tuple(offset, binsha, 0)"""
     return unpack_from(">L20s", self._cursor.map(), 1024 + i * 24) + (0, )
Exemple #20
0
 def _crc_v2(self, i):
     """:return: 4 bytes crc for the object at index i"""
     return unpack_from(">L", self._cursor.map(),
                        self._crc_list_offset + i * 4)[0]