Ejemplo n.º 1
0
 def test_compute_file_sha(self):
     f = BytesIO('abcd1234wxyz')
     self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
                      compute_file_sha(f).hexdigest())
     self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
                      compute_file_sha(f, buffer_size=5).hexdigest())
     self.assertEqual(sha1('abcd1234').hexdigest(),
                      compute_file_sha(f, end_ofs=-4).hexdigest())
     self.assertEqual(sha1('1234wxyz').hexdigest(),
                      compute_file_sha(f, start_ofs=4).hexdigest())
     self.assertEqual(
       sha1('1234').hexdigest(),
       compute_file_sha(f, start_ofs=4, end_ofs=-4).hexdigest())
Ejemplo n.º 2
0
 def test_compute_file_sha(self):
     f = BytesIO('abcd1234wxyz')
     self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
                      compute_file_sha(f).hexdigest())
     self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
                      compute_file_sha(f, buffer_size=5).hexdigest())
     self.assertEqual(sha1('abcd1234').hexdigest(),
                      compute_file_sha(f, end_ofs=-4).hexdigest())
     self.assertEqual(sha1('1234wxyz').hexdigest(),
                      compute_file_sha(f, start_ofs=4).hexdigest())
     self.assertEqual(
       sha1('1234').hexdigest(),
       compute_file_sha(f, start_ofs=4, end_ofs=-4).hexdigest())
Ejemplo n.º 3
0
 def test_compute_file_sha(self):
     with BytesIO(b'abcd1234wxyz') as f:
         self.assertEqual(hashlib.sha1(b'abcd1234wxyz').hexdigest(),
                          compute_file_sha(f).hexdigest())
         self.assertEqual(hashlib.sha1(b'abcd1234wxyz').hexdigest(),
                          compute_file_sha(f, buffer_size=5).hexdigest())
         self.assertEqual(hashlib.sha1(b'abcd1234').hexdigest(),
                          compute_file_sha(f, end_ofs=-4).hexdigest())
         self.assertEqual(hashlib.sha1(b'1234wxyz').hexdigest(),
                          compute_file_sha(f, start_ofs=4).hexdigest())
         self.assertEqual(
           hashlib.sha1(b'1234').hexdigest(),
           compute_file_sha(f, start_ofs=4, end_ofs=-4).hexdigest())
Ejemplo n.º 4
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.flush()

        # Move the pack in.
        entries.sort()
        pack_base_name = posixpath.join(
            self.pack_dir,
            "pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)),
        )
        self.scon.put_object(pack_base_name + ".pack", f)

        # Write the index.
        filename = pack_base_name + ".idx"
        index_file = BytesIO()
        write_pack_index_v2(index_file, entries, pack_sha)
        self.scon.put_object(filename, index_file)

        # Write pack info.
        f.seek(0)
        pack_data = PackData(filename="", file=f)
        index_file.seek(0)
        pack_index = load_pack_index_file("", index_file)
        serialized_pack_info = pack_info_create(pack_data, pack_index)
        f.close()
        index_file.close()
        pack_info_file = BytesIO(serialized_pack_info)
        filename = pack_base_name + ".info"
        self.scon.put_object(filename, pack_info_file)
        pack_info_file.close()

        # Add the pack to the store and return it.
        final_pack = SwiftPack(pack_base_name, scon=self.scon)
        final_pack.check_length_and_checksum()
        self._add_cached_pack(pack_base_name, final_pack)
        return final_pack
    def _complete_thin_pack(self, f, path, copier, indexer):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param f: Open file object for the pack.
        :param path: Path to the pack file.
        :param copier: A PackStreamCopier to use for writing pack data.
        :param indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.close()

        # Move the pack in.
        entries.sort()
        pack_base_name = self._get_pack_basepath(entries)
        try:
            os.rename(path, pack_base_name + '.pack')
        except WindowsError:
            os.remove(pack_base_name + '.pack')
            os.rename(path, pack_base_name + '.pack')

        # Write the index.
        index_file = GitFile(pack_base_name + '.idx', 'wb')
        try:
            write_pack_index_v2(index_file, entries, pack_sha)
            index_file.close()
        finally:
            index_file.abort()

        # Add the pack to the store and return it.
        final_pack = Pack(pack_base_name)
        final_pack.check_length_and_checksum()
        self._add_known_pack(pack_base_name, final_pack)
        return final_pack
Ejemplo n.º 6
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.flush()

        # Move the pack in.
        entries.sort()
        pack_base_name = posixpath.join(
            self.pack_dir,
            'pack-' + iter_sha1(e[0] for e in entries).decode(
                sys.getfilesystemencoding()))
        self.scon.put_object(pack_base_name + '.pack', f)

        # Write the index.
        filename = pack_base_name + '.idx'
        index_file = BytesIO()
        write_pack_index_v2(index_file, entries, pack_sha)
        self.scon.put_object(filename, index_file)

        # Write pack info.
        f.seek(0)
        pack_data = PackData(filename="", file=f)
        index_file.seek(0)
        pack_index = load_pack_index_file('', index_file)
        serialized_pack_info = pack_info_create(pack_data, pack_index)
        f.close()
        index_file.close()
        pack_info_file = BytesIO(serialized_pack_info)
        filename = pack_base_name + '.info'
        self.scon.put_object(filename, pack_info_file)
        pack_info_file.close()

        # Add the pack to the store and return it.
        final_pack = SwiftPack(pack_base_name, scon=self.scon)
        final_pack.check_length_and_checksum()
        self._add_cached_pack(pack_base_name, final_pack)
        return final_pack
Ejemplo n.º 7
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param f: Open file object for the pack.
        :param path: Path to the pack file.
        :param copier: A PackStreamCopier to use for writing pack data.
        :param indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.close()

        # Move the pack in.
        entries.sort()
        pack_base_name = os.path.join(
          self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
        os.rename(path, pack_base_name + '.pack')

        # Write the index.
        index_file = GitFile(pack_base_name + '.idx', 'wb')
        try:
            write_pack_index_v2(index_file, entries, pack_sha)
            index_file.close()
        finally:
            index_file.abort()

        # Add the pack to the store and return it.
        final_pack = Pack(pack_base_name)
        final_pack.check_length_and_checksum()
        self._add_known_pack(final_pack)
        return final_pack
Ejemplo n.º 8
0
    def from_thinpack(cls, pack_store, f, indexer, resolve_ext_ref):
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = resolve_ext_ref(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        #f.close()

        #write the pack
        blob_name = files.blobstore.create(
            mime_type='application/octet-stream')
        with files.open(blob_name, 'a') as blob:
            blob.write(f.getvalue())
        files.finalize(blob_name)

        #store pack info
        pack_store.data = files.blobstore.get_blob_key(blob_name)
        #pack_store.sha1 #sha's are added when writing the index
        pack_store.size = f.tell()
        pack_store.checksum = sha_to_hex(pack_sha)
        pack_store.save()

        # Write the index.
        pack_indexes = [pack_store]
        for (name, offset, entry_checksum) in entries:
            idx = PackStoreIndex(packref=pack_store,
                                 sha=sha_to_hex(name),
                                 offset=offset,
                                 crc32=entry_checksum)
            pack_store.sha1.append(sha_to_hex(name))
            pack_indexes.append(idx)
        db.save(pack_indexes)

        # Add the pack to the store and return it.
        final_pack = Pack(pack_store)
        final_pack.check_length_and_checksum()
        return final_pack
Ejemplo n.º 9
0
    def from_thinpack(cls, pack_store, f, indexer, resolve_ext_ref):
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = resolve_ext_ref(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        # f.close()

        # write the pack
        blob_name = files.blobstore.create(mime_type="application/octet-stream")
        with files.open(blob_name, "a") as blob:
            blob.write(f.getvalue())
        files.finalize(blob_name)

        # store pack info
        pack_store.data = files.blobstore.get_blob_key(blob_name)
        # pack_store.sha1 #sha's are added when writing the index
        pack_store.size = f.tell()
        pack_store.checksum = sha_to_hex(pack_sha)
        pack_store.save()

        # Write the index.
        pack_indexes = [pack_store]
        for (name, offset, entry_checksum) in entries:
            idx = PackStoreIndex(packref=pack_store, sha=sha_to_hex(name), offset=offset, crc32=entry_checksum)
            pack_store.sha1.append(sha_to_hex(name))
            pack_indexes.append(idx)
        db.save(pack_indexes)

        # Add the pack to the store and return it.
        final_pack = Pack(pack_store)
        final_pack.check_length_and_checksum()
        return final_pack
Ejemplo n.º 10
0
    def _complete_thin_pack(self, f, indexer):
        """Complete a thin pack by adding external references.

        :param f: Open file object for the pack.
        :param indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            write_pack_object(f, type_num, data, sha=new_sha)
        pack_sha = new_sha.digest()
        f.write(pack_sha)
Ejemplo n.º 11
0
    def _complete_thin_pack(self, f, indexer):
        """Complete a thin pack by adding external references.

        :param f: Open file object for the pack.
        :param indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            write_pack_object(f, type_num, data, sha=new_sha)
        pack_sha = new_sha.digest()
        f.write(pack_sha)
Ejemplo n.º 12
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        """Move a specific file containing a pack into the pack directory.

        Note: The file should be on the same file system as the
            packs directory.

        Args:
          f: Open file object for the pack.
          path: Path to the pack file.
          copier: A PackStreamCopier to use for writing pack data.
          indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(
                f,
                type_num,
                data,
                sha=new_sha,
                compression_level=self.pack_compression_level,
            )
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.close()

        # Move the pack in.
        entries.sort()
        pack_base_name = self._get_pack_basepath(entries)
        target_pack = pack_base_name + ".pack"
        if sys.platform == "win32":
            # Windows might have the target pack file lingering. Attempt
            # removal, silently passing if the target does not exist.
            try:
                os.remove(target_pack)
            except FileNotFoundError:
                pass
        os.rename(path, target_pack)

        # Write the index.
        index_file = GitFile(pack_base_name + ".idx", "wb")
        try:
            write_pack_index_v2(index_file, entries, pack_sha)
            index_file.close()
        finally:
            index_file.abort()

        # Add the pack to the store and return it.
        final_pack = Pack(pack_base_name)
        final_pack.check_length_and_checksum()
        self._add_cached_pack(pack_base_name, final_pack)
        return final_pack
Ejemplo n.º 13
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param f: Open file object for the pack.
        :param path: Path to the pack file.
        :param copier: A PackStreamCopier to use for writing pack data.
        :param indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.close()

        # Move the pack in.
        entries.sort()
        pack_base_name = self._get_pack_basepath(entries)
        target_pack = pack_base_name + '.pack'
        if sys.platform == 'win32':
            # Windows might have the target pack file lingering. Attempt
            # removal, silently passing if the target does not exist.
            try:
                os.remove(target_pack)
            except (IOError, OSError) as e:
                if e.errno != errno.ENOENT:
                    raise
        os.rename(path, target_pack)

        # Write the index.
        index_file = GitFile(pack_base_name + '.idx', 'wb')
        try:
            write_pack_index_v2(index_file, entries, pack_sha)
            index_file.close()
        finally:
            index_file.abort()

        # Add the pack to the store and return it.
        final_pack = Pack(pack_base_name)
        final_pack.check_length_and_checksum()
        self._add_cached_pack(pack_base_name, final_pack)
        return final_pack