Exemplo n.º 1
0
    def move_in_pack(self, path):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param path: Path to the pack file.
        """
        p = PackData(path)
        try:
            entries = p.sorted_entries()
            basename = os.path.join(
                self.pack_dir,
                "pack-%s" % iter_sha1(entry[0] for entry in entries))
            f = GitFile(basename + ".idx", "wb")
            try:
                write_pack_index_v2(f, entries, p.get_stored_checksum())
            finally:
                f.close()
        finally:
            p.close()
        os.rename(path, basename + ".pack")
        final_pack = Pack(basename)
        self._add_known_pack(final_pack)
        return final_pack
Exemplo n.º 2
0
        def commit():
            if pf.tell() == 0:
                pf.close()
                return None

            pf.seek(0)
            p = PackData(pf.name, pf)
            entries = p.sorted_entries()
            basename = iter_sha1(entry[0] for entry in entries).decode('ascii')
            idxf = tempfile.SpooledTemporaryFile()
            checksum = p.get_stored_checksum()
            write_pack_index_v2(idxf, entries, checksum)
            idxf.seek(0)
            idx = load_pack_index_file(basename + '.idx', idxf)
            for pack in self.packs:
                if pack.get_stored_checksum() == p.get_stored_checksum():
                    p.close()
                    idx.close()
                    return pack
            pf.seek(0)
            idxf.seek(0)
            self._upload_pack(basename, pf, idxf)
            final_pack = Pack.from_objects(p, idx)
            self._add_cached_pack(basename, final_pack)
            return final_pack
Exemplo n.º 3
0
	def upload_pack_file(self, path):
		p = PackData(path)
		entries = p.sorted_entries()

		# get the sha1 of the pack, same method as dulwich's move_in_pack()
		pack_sha = iter_sha1(e[0] for e in entries)
		key_prefix = calc_pack_prefix(self.prefix, pack_sha)
		pack_key_name = '%s.pack' % key_prefix

		# FIXME: LOCK HERE? Possibly different pack files could
		#        have the same shas, depending on compression?

		log.debug('Uploading %s to %s' % (path, pack_key_name))

		pack_key = self.bucket.new_key(pack_key_name)
		pack_key.set_contents_from_filename(path)
		index_key_name = '%s.idx' % key_prefix

		index_key = self.bucket.new_key(index_key_name)

		index_fd, index_path = tempfile.mkstemp(suffix = '.idx')
		try:
			f = os.fdopen(index_fd, 'wb')
			write_pack_index_v2(f, entries, p.get_stored_checksum())
			os.fsync(index_fd)
			f.close()

			log.debug('Uploading %s to %s' % (index_path, index_key_name))
			index_key.set_contents_from_filename(index_path)
		finally:
			os.remove(index_path)

		p.close()

		return self._create_pack(key_prefix)
Exemplo n.º 4
0
    def move_in_pack(self, f):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param path: Path to the pack file.
        """
        f.seek(0)
        p = PackData("", f, len(f.getvalue()))
        entries = p.sorted_entries()
        basename = "pack-%s" % iter_sha1(entry[0]
                                         for entry in entries).decode('ascii')
        p._filename = basename + ".pack"
        f.seek(0)
        self.pack_transport.put_file(basename + ".pack", f)
        idxfile = self.pack_transport.open_write_stream(basename + ".idx")
        try:
            write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
        finally:
            idxfile.close()
        idxfile = self.pack_transport.get(basename + ".idx")
        idx = load_pack_index_file(basename + ".idx", idxfile)
        final_pack = Pack.from_objects(p, idx)
        final_pack._basename = basename
        self._add_known_pack(basename, final_pack)
        return final_pack
Exemplo n.º 5
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.flush()

        # Move the pack in.
        entries.sort()
        pack_base_name = posixpath.join(
            self.pack_dir,
            "pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)),
        )
        self.scon.put_object(pack_base_name + ".pack", f)

        # Write the index.
        filename = pack_base_name + ".idx"
        index_file = BytesIO()
        write_pack_index_v2(index_file, entries, pack_sha)
        self.scon.put_object(filename, index_file)

        # Write pack info.
        f.seek(0)
        pack_data = PackData(filename="", file=f)
        index_file.seek(0)
        pack_index = load_pack_index_file("", index_file)
        serialized_pack_info = pack_info_create(pack_data, pack_index)
        f.close()
        index_file.close()
        pack_info_file = BytesIO(serialized_pack_info)
        filename = pack_base_name + ".info"
        self.scon.put_object(filename, pack_info_file)
        pack_info_file.close()

        # Add the pack to the store and return it.
        final_pack = SwiftPack(pack_base_name, scon=self.scon)
        final_pack.check_length_and_checksum()
        self._add_cached_pack(pack_base_name, final_pack)
        return final_pack
Exemplo n.º 6
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.flush()

        # Move the pack in.
        entries.sort()
        pack_base_name = posixpath.join(
            self.pack_dir,
            'pack-' + iter_sha1(e[0] for e in entries).decode(
                sys.getfilesystemencoding()))
        self.scon.put_object(pack_base_name + '.pack', f)

        # Write the index.
        filename = pack_base_name + '.idx'
        index_file = BytesIO()
        write_pack_index_v2(index_file, entries, pack_sha)
        self.scon.put_object(filename, index_file)

        # Write pack info.
        f.seek(0)
        pack_data = PackData(filename="", file=f)
        index_file.seek(0)
        pack_index = load_pack_index_file('', index_file)
        serialized_pack_info = pack_info_create(pack_data, pack_index)
        f.close()
        index_file.close()
        pack_info_file = BytesIO(serialized_pack_info)
        filename = pack_base_name + '.info'
        self.scon.put_object(filename, pack_info_file)
        pack_info_file.close()

        # Add the pack to the store and return it.
        final_pack = SwiftPack(pack_base_name, scon=self.scon)
        final_pack.check_length_and_checksum()
        self._add_cached_pack(pack_base_name, final_pack)
        return final_pack
Exemplo n.º 7
0
 def compute_blob_hash(fname, chunk_size=1<<16):
     obj_len = os.path.getsize(fname)
     head = object_header(Blob.type_num, obj_len)
     with open(fname, "rb") as f:
         def read_chunk():
             return f.read(chunk_size)
         my_iter = chain([head], iter(read_chunk,''))
         return iter_sha1(my_iter)
Exemplo n.º 8
0
 def compute_blob_hash(fname, chunk_size=1<<16):
     obj_len = os.path.getsize(fname)
     head = object_header(Blob.type_num, obj_len)
     with open(fname, "rb") as f:
         def read_chunk():
             return f.read(chunk_size)
         my_iter = chain([head], iter(read_chunk,''))
         return iter_sha1(my_iter)
Exemplo n.º 9
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param f: Open file object for the pack.
        :param path: Path to the pack file.
        :param copier: A PackStreamCopier to use for writing pack data.
        :param indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.close()

        # Move the pack in.
        entries.sort()
        pack_base_name = os.path.join(
          self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
        os.rename(path, pack_base_name + '.pack')

        # Write the index.
        index_file = GitFile(pack_base_name + '.idx', 'wb')
        try:
            write_pack_index_v2(index_file, entries, pack_sha)
            index_file.close()
        finally:
            index_file.abort()

        # Add the pack to the store and return it.
        final_pack = Pack(pack_base_name)
        final_pack.check_length_and_checksum()
        self._add_known_pack(final_pack)
        return final_pack
Exemplo n.º 10
0
    def _complete_thin_pack(self, f, path, copier, indexer):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param f: Open file object for the pack.
        :param path: Path to the pack file.
        :param copier: A PackStreamCopier to use for writing pack data.
        :param indexer: A PackIndexer for indexing the pack.
        """
        entries = list(indexer)

        # Update the header with the new number of objects.
        f.seek(0)
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))

        # Must flush before reading (http://bugs.python.org/issue3207)
        f.flush()

        # Rescan the rest of the pack, computing the SHA with the new header.
        new_sha = compute_file_sha(f, end_ofs=-20)

        # Must reposition before writing (http://bugs.python.org/issue3207)
        f.seek(0, os.SEEK_CUR)

        # Complete the pack.
        for ext_sha in indexer.ext_refs():
            assert len(ext_sha) == 20
            type_num, data = self.get_raw(ext_sha)
            offset = f.tell()
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
            entries.append((ext_sha, offset, crc32))
        pack_sha = new_sha.digest()
        f.write(pack_sha)
        f.close()

        # Move the pack in.
        entries.sort()
        pack_base_name = os.path.join(
          self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
        os.rename(path, pack_base_name + '.pack')

        # Write the index.
        index_file = GitFile(pack_base_name + '.idx', 'wb')
        try:
            write_pack_index_v2(index_file, entries, pack_sha)
            index_file.close()
        finally:
            index_file.abort()

        # Add the pack to the store and return it.
        final_pack = Pack(pack_base_name)
        final_pack.check_length_and_checksum()
        self._add_known_pack(pack_base_name, final_pack)
        return final_pack
Exemplo n.º 11
0
    def move_in_pack(self, path):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the 
            packs directory.

        :param path: Path to the pack file.
        """
        p = PackData(path)
        entries = p.sorted_entries()
        basename = os.path.join(self.pack_dir, 
            "pack-%s" % iter_sha1(entry[0] for entry in entries))
        write_pack_index_v2(basename+".idx", entries, p.get_stored_checksum())
        os.rename(path, basename + ".pack")
        self._add_known_pack(basename)
Exemplo n.º 12
0
    def move_in_pack(self, path):
        """Move a specific file containing a pack into the pack directory.

        :note: The file should be on the same file system as the
            packs directory.

        :param path: Path to the pack file.
        """
        with PackData(path) as p:
            entries = p.sorted_entries()
            basename = os.path.join(self.pack_dir,
                'pack-' + str(iter_sha1(entry[0].bytes for entry in entries)))
            with GitFile(basename+".idx", "wb") as f:
                write_pack_index_v2(f, entries, p.get_stored_checksum())
        os.rename(path, basename + ".pack")
        final_pack = Pack(basename)
        self._add_known_pack(final_pack)
        return final_pack
Exemplo n.º 13
0
 def commit():
     f.seek(0)
     pack = PackData(file=f, filename="")
     entries = pack.sorted_entries()
     if len(entries):
         basename = posixpath.join(
             self.pack_dir,
             "pack-%s" % iter_sha1(entry[0] for entry in entries))
         index = BytesIO()
         write_pack_index_v2(index, entries, pack.get_stored_checksum())
         self.scon.put_object(basename + ".pack", f)
         f.close()
         self.scon.put_object(basename + ".idx", index)
         index.close()
         final_pack = SwiftPack(basename, scon=self.scon)
         final_pack.check_length_and_checksum()
         self._add_known_pack(basename, final_pack)
         return final_pack
     else:
         return None
Exemplo n.º 14
0
 def commit():
     f.seek(0)
     pack = PackData(file=f, filename="")
     entries = pack.sorted_entries()
     if len(entries):
         basename = posixpath.join(self.pack_dir,
                                   "pack-%s" %
                                   iter_sha1(entry[0] for
                                             entry in entries))
         index = BytesIO()
         write_pack_index_v2(index, entries, pack.get_stored_checksum())
         self.scon.put_object(basename + ".pack", f)
         f.close()
         self.scon.put_object(basename + ".idx", index)
         index.close()
         final_pack = SwiftPack(basename, scon=self.scon)
         final_pack.check_length_and_checksum()
         self._add_known_pack(basename, final_pack)
         return final_pack
     else:
         return None
Exemplo n.º 15
0
 def _get_pack_basepath(self, entries):
     suffix = iter_sha1(entry[0] for entry in entries)
     # TODO: Handle self.pack_dir being bytes
     suffix = suffix.decode("ascii")
     return os.path.join(self.pack_dir, "pack-" + suffix)
Exemplo n.º 16
0
 def _get_pack_basepath(self, entries):
     suffix = iter_sha1(entry[0] for entry in entries)
     # TODO: Handle self.pack_dir being bytes
     suffix = suffix.decode('ascii')
     return os.path.join(self.pack_dir, "pack-" + suffix)
Exemplo n.º 17
0
 def _get_pack_basepath(self, entries):
     suffix = iter_sha1(entry[0] for entry in entries)
     # TODO: Handle self.pack_dir being bytes
     suffix = suffix.decode('ascii')
     return self.pack_transport.local_abspath("pack-" + suffix)