def _update_pack_cache(self): pack_files = set() pack_dir_contents = self._pack_names() for name in pack_dir_contents: if name.startswith("pack-") and name.endswith(".pack"): # verify that idx exists first (otherwise the pack was not yet # fully written) idx_name = os.path.splitext(name)[0] + ".idx" if idx_name in pack_dir_contents: pack_files.add(os.path.splitext(name)[0]) new_packs = [] for basename in pack_files: pack_name = basename + ".pack" if basename not in self._pack_cache: try: size = self.pack_transport.stat(pack_name).st_size except TransportNotPossible: f = self.pack_transport.get(pack_name) pd = PackData(pack_name, f) else: pd = PackData(pack_name, self.pack_transport.get(pack_name), size=size) idxname = basename + ".idx" idx = load_pack_index_file(idxname, self.pack_transport.get(idxname)) pack = Pack.from_objects(pd, idx) pack._basename = basename self._pack_cache[basename] = pack new_packs.append(pack) # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close() return new_packs
def _update_pack_cache(self): pack_files = set(self._pack_names()) new_packs = [] for basename in pack_files: pack_name = basename + ".pack" if basename not in self._pack_cache: try: size = self.pack_transport.stat(pack_name).st_size except TransportNotPossible: f = self.pack_transport.get(pack_name) # TODO(jelmer): Don't read entire file into memory? f = BytesIO(f.read()) pd = PackData(pack_name, f) else: pd = PackData( pack_name, self.pack_transport.get(pack_name), size=size) idxname = basename + ".idx" idx = load_pack_index_file( idxname, self.pack_transport.get(idxname)) pack = Pack.from_objects(pd, idx) pack._basename = basename self._pack_cache[basename] = pack new_packs.append(pack) # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close() return new_packs
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. Note: The file should be on the same file system as the packs directory. Args: path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) index_name = basename + ".idx" if not os.path.exists(index_name): with GitFile(index_name, "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) for pack in self.packs: if pack._basename == basename: return pack target_pack = basename + ".pack" if sys.platform == "win32": # Windows might have the target pack file lingering. Attempt # removal, silently passing if the target does not exist. try: os.remove(target_pack) except FileNotFoundError: pass os.rename(path, target_pack) final_pack = Pack(basename) self._add_cached_pack(basename, final_pack) return final_pack
def setUp(self): super(TestThinPack, self).setUp() self.store = MemoryObjectStore() self.blobs = {} for blob in (b'foo', b'bar', b'foo1234', b'bar2468'): self.blobs[blob] = make_object(Blob, data=blob) self.store.add_object(self.blobs[b'foo']) self.store.add_object(self.blobs[b'bar']) # Build a thin pack. 'foo' is as an external reference, 'bar' an # internal reference. self.pack_dir = tempfile.mkdtemp() if not isinstance(self.pack_dir, bytes): self.pack_dir = self.pack_dir.encode(sys.getfilesystemencoding()) self.addCleanup(shutil.rmtree, self.pack_dir) self.pack_prefix = os.path.join(self.pack_dir, b'pack') with open(self.pack_prefix + b'.pack', 'wb') as f: build_pack(f, [ (REF_DELTA, (self.blobs[b'foo'].id, b'foo1234')), (Blob.type_num, b'bar'), (REF_DELTA, (self.blobs[b'bar'].id, b'bar2468'))], store=self.store) # Index the new pack. with self.make_pack(True) as pack: with PackData(pack._data_path) as data: data.pack = pack data.create_index(self.pack_prefix + b'.idx') del self.store[self.blobs[b'bar'].id]
def setUp(self): super(TestThinPack, self).setUp() self.store = MemoryObjectStore() self.blobs = {} for blob in ('foo', 'bar', 'foo1234', 'bar2468'): self.blobs[blob] = make_object(Blob, data=blob) self.store.add_object(self.blobs['foo']) self.store.add_object(self.blobs['bar']) # Build a thin pack. 'foo' is as an external reference, 'bar' an # internal reference. self.pack_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.pack_dir) self.pack_prefix = os.path.join(self.pack_dir, 'pack') f = open(self.pack_prefix + '.pack', 'wb') try: build_pack(f, [ (REF_DELTA, (self.blobs['foo'].id, 'foo1234')), (Blob.type_num, 'bar'), (REF_DELTA, (self.blobs['bar'].id, 'bar2468'))], store=self.store) finally: f.close() # Index the new pack. with self.make_pack(True) as pack: with PackData(pack._data_path) as data: data.pack = pack data.create_index(self.pack_prefix + '.idx') del self.store[self.blobs['bar'].id]
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ p = PackData(path) try: entries = p.sorted_entries() basename = os.path.join( self.pack_dir, "pack-%s" % iter_sha1(entry[0] for entry in entries)) f = GitFile(basename + ".idx", "wb") try: write_pack_index_v2(f, entries, p.get_stored_checksum()) finally: f.close() finally: p.close() os.rename(path, basename + ".pack") final_pack = Pack(basename) self._add_known_pack(final_pack) return final_pack
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + ".idx", "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) if self._pack_cache is None or self._pack_cache_stale(): self._update_pack_cache() try: return self._pack_cache[basename] except KeyError: pass else: os.unlink(path) try: os.rename(path, basename + ".pack") except OSError as e: if e.errno == errno.EEXIST: # This can happen on Windows.. # It's safe to ignore this, since if the file already exists, # it should have the same contents as the one we just # generated. os.unlink(path) else: raise final_pack = Pack(basename) self._add_known_pack(basename, final_pack) return final_pack
def commit(): if pf.tell() == 0: pf.close() return None pf.seek(0) p = PackData(pf.name, pf) entries = p.sorted_entries() basename = iter_sha1(entry[0] for entry in entries).decode('ascii') idxf = tempfile.SpooledTemporaryFile() checksum = p.get_stored_checksum() write_pack_index_v2(idxf, entries, checksum) idxf.seek(0) idx = load_pack_index_file(basename + '.idx', idxf) for pack in self.packs: if pack.get_stored_checksum() == p.get_stored_checksum(): p.close() idx.close() return pack pf.seek(0) idxf.seek(0) self._upload_pack(basename, pf, idxf) final_pack = Pack.from_objects(p, idx) self._add_cached_pack(basename, final_pack) return final_pack
def setUp(self): super(TestThinPack, self).setUp() self.store = MemoryObjectStore() self.blobs = {} for blob in (b"foo", b"bar", b"foo1234", b"bar2468"): self.blobs[blob] = make_object(Blob, data=blob) self.store.add_object(self.blobs[b"foo"]) self.store.add_object(self.blobs[b"bar"]) # Build a thin pack. 'foo' is as an external reference, 'bar' an # internal reference. self.pack_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.pack_dir) self.pack_prefix = os.path.join(self.pack_dir, "pack") with open(self.pack_prefix + ".pack", "wb") as f: build_pack( f, [ (REF_DELTA, (self.blobs[b"foo"].id, b"foo1234")), (Blob.type_num, b"bar"), (REF_DELTA, (self.blobs[b"bar"].id, b"bar2468")), ], store=self.store, ) # Index the new pack. with self.make_pack(True) as pack: with PackData(pack._data_path) as data: data.pack = pack data.create_index(self.pack_prefix + ".idx") del self.store[self.blobs[b"bar"].id]
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + ".idx", "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) if self._pack_cache is None or self._pack_cache_stale(): self._update_pack_cache() try: return self._pack_cache[basename] except KeyError: pass else: os.unlink(path) os.rename(path, basename + ".pack") final_pack = Pack(basename) self._add_known_pack(basename, final_pack) return final_pack
def move_in_pack(self, f): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ f.seek(0) p = PackData("", f, len(f.getvalue())) entries = p.sorted_entries() basename = "pack-%s" % iter_sha1(entry[0] for entry in entries).decode('ascii') p._filename = basename + ".pack" f.seek(0) self.pack_transport.put_file(basename + ".pack", f) idxfile = self.pack_transport.open_write_stream(basename + ".idx") try: write_pack_index_v2(idxfile, entries, p.get_stored_checksum()) finally: idxfile.close() idxfile = self.pack_transport.get(basename + ".idx") idx = load_pack_index_file(basename + ".idx", idxfile) final_pack = Pack.from_objects(p, idx) final_pack._basename = basename self._add_known_pack(basename, final_pack) return final_pack
def move_in_thin_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ data = PackData(path) # Write index for the thin pack (do we really need this?) temppath = os.path.join(self.pack_dir, sha_to_hex(urllib2.randombytes(20))+".tempidx") data.create_index_v2(temppath, self.get_raw) p = Pack.from_objects(data, load_pack_index(temppath)) # Write a full pack version temppath = os.path.join(self.pack_dir, sha_to_hex(urllib2.randombytes(20))+".temppack") write_pack(temppath, ((o, None) for o in p.iterobjects(self.get_raw)), len(p)) pack_sha = load_pack_index(temppath+".idx").objects_sha1() newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha) os.rename(temppath+".pack", newbasename+".pack") os.rename(temppath+".idx", newbasename+".idx") self._add_known_pack(newbasename)
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + ".idx", "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) for pack in self.packs: if pack._basename == basename: return pack target_pack = basename + '.pack' if sys.platform == 'win32': # Windows might have the target pack file lingering. Attempt # removal, silently passing if the target does not exist. try: os.remove(target_pack) except (IOError, OSError) as e: if e.errno != errno.ENOENT: raise os.rename(path, target_pack) final_pack = Pack(basename) self._add_cached_pack(basename, final_pack) return final_pack
def make_pack_iter(self, f, thin=None): if thin is None: thin = bool(list(self.store)) resolve_ext_ref = thin and self.get_raw_no_repeat or None data = PackData('test.pack', file=f) return TestPackIterator.for_pack_data( data, resolve_ext_ref=resolve_ext_ref)
def _complete_thin_pack(self, f, path, copier, indexer): entries = list(indexer) # Update the header with the new number of objects. f.seek(0) write_pack_header(f, len(entries) + len(indexer.ext_refs())) # Must flush before reading (http://bugs.python.org/issue3207) f.flush() # Rescan the rest of the pack, computing the SHA with the new header. new_sha = compute_file_sha(f, end_ofs=-20) # Must reposition before writing (http://bugs.python.org/issue3207) f.seek(0, os.SEEK_CUR) # Complete the pack. for ext_sha in indexer.ext_refs(): assert len(ext_sha) == 20 type_num, data = self.get_raw(ext_sha) offset = f.tell() crc32 = write_pack_object(f, type_num, data, sha=new_sha) entries.append((ext_sha, offset, crc32)) pack_sha = new_sha.digest() f.write(pack_sha) f.flush() # Move the pack in. entries.sort() pack_base_name = posixpath.join( self.pack_dir, "pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)), ) self.scon.put_object(pack_base_name + ".pack", f) # Write the index. filename = pack_base_name + ".idx" index_file = BytesIO() write_pack_index_v2(index_file, entries, pack_sha) self.scon.put_object(filename, index_file) # Write pack info. f.seek(0) pack_data = PackData(filename="", file=f) index_file.seek(0) pack_index = load_pack_index_file("", index_file) serialized_pack_info = pack_info_create(pack_data, pack_index) f.close() index_file.close() pack_info_file = BytesIO(serialized_pack_info) filename = pack_base_name + ".info" self.scon.put_object(filename, pack_info_file) pack_info_file.close() # Add the pack to the store and return it. final_pack = SwiftPack(pack_base_name, scon=self.scon) final_pack.check_length_and_checksum() self._add_cached_pack(pack_base_name, final_pack) return final_pack
def fetch_objects(self, determine_wants, graph_walker, progress=None): fd, path = tempfile.mkstemp(suffix=".pack") self.fetch_pack(determine_wants, graph_walker, lambda x: os.write(fd, x), progress) os.close(fd) basename = path[:-len(".pack")] p = PackData(path) p.create_index_v2(basename + ".idx") pack = Pack(basename) os.remove(path) return (len(p), pack.iterobjects())
def _load_packs(self): ret = [] for name in self._pack_names(): if name.startswith("pack-") and name.endswith(".pack"): try: size = self.pack_transport.stat(name).st_size except TransportNotPossible: f = self.pack_transport.get(name) pd = PackData(name, f) else: pd = PackData(name, self.pack_transport.get(name), size=size) idxname = name.replace(".pack", ".idx") idx = load_pack_index_file(idxname, self.pack_transport.get(idxname)) pack = Pack.from_objects(pd, idx) pack._basename = idxname[:-4] ret.append(pack) return ret
def test_checksum_mismatch(self): with self.get_pack_data(pack1_sha) as data: index = self.get_pack_index(pack1_sha) Pack.from_objects(data, index).check_length_and_checksum() data._file.seek(0) bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20)) bad_data = PackData('', file=bad_file) bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index) self.assertRaises(ChecksumMismatch, lambda: bad_pack.data) self.assertRaises(ChecksumMismatch, lambda: bad_pack.check_length_and_checksum())
def test_length_mismatch(self): with self.get_pack_data(pack1_sha) as data: index = self.get_pack_index(pack1_sha) Pack.from_objects(data, index).check_length_and_checksum() data._file.seek(12) bad_file = BytesIO() write_pack_header(bad_file, 9999) bad_file.write(data._file.read()) bad_file = BytesIO(bad_file.getvalue()) bad_data = PackData('', file=bad_file) bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index) self.assertRaises(AssertionError, lambda: bad_pack.data) self.assertRaises(AssertionError, lambda: bad_pack.check_length_and_checksum())
def apply_pack(self, refs, read): """ apply pack from client to current repository """ fd, path = tempfile.mkstemp(suffix=".pack") f = os.fdopen(fd, 'w') f.write(read()) f.close() p = PackData(path) entries = p.sorted_entries() write_pack_index_v2(path[:-5] + ".idx", entries, p.calculate_checksum()) def get_objects(): pack = Pack(path[:-5]) for obj in pack.iterobjects(): yield obj target = Repository.open(self.directory) target.lock_write() try: target.start_write_group() try: import_git_objects(target, self.mapping, iter(get_objects())) finally: target.commit_write_group() finally: target.unlock() for oldsha, sha, ref in refs: if ref[:11] == 'refs/heads/': branch_nick = ref[11:] try: target_dir = BzrDir.open(self.directory + "/" + branch_nick) except: target_dir = BzrDir.create(self.directory + "/" + branch_nick) try: target_branch = target_dir.open_branch() except: target_branch = target_dir.create_branch() rev_id = self.mapping.revision_id_foreign_to_bzr(sha) target_branch.generate_revision_history(rev_id)
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + b'.idx', "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) os.rename(path, basename + b'.pack') final_pack = Pack(basename) self._add_known_pack(basename, final_pack) return final_pack
def commit(): f.seek(0) pack = PackData(file=f, filename="") entries = pack.sorted_entries() if len(entries): basename = posixpath.join( self.pack_dir, "pack-%s" % iter_sha1(entry[0] for entry in entries)) index = BytesIO() write_pack_index_v2(index, entries, pack.get_stored_checksum()) self.scon.put_object(basename + ".pack", f) f.close() self.scon.put_object(basename + ".idx", index) index.close() final_pack = SwiftPack(basename, scon=self.scon) final_pack.check_length_and_checksum() self._add_known_pack(basename, final_pack) return final_pack else: return None
def commit(): from .fetch import import_git_objects os.fsync(fd) f.close() if os.path.getsize(path) == 0: return pd = PackData(path) pd.create_index_v2(path[:-5] + ".idx", self.object_store.get_raw) p = Pack(path[:-5]) with self.repository.lock_write(): self.repository.start_write_group() try: import_git_objects(self.repository, self.mapping, p.iterobjects(get_raw=self.get_raw), self.object_store) except BaseException: self.repository.abort_write_group() raise else: self.repository.commit_write_group()
def get_pack_data(self, sha): """Returns a PackData object from the datadir with the given sha""" return PackData( os.path.join(self.datadir, "pack-%s.pack" % sha.decode("ascii")))
def _pack_data() -> PackData: pack_data_bytes = self._http_get(f"objects/pack/{pack_name}") return PackData(pack_name, file=pack_data_bytes)
def get_pack_data(self, sha): """Returns a PackData object from the datadir with the given sha""" return PackData(os.path.join(self.datadir, 'pack-%s.pack' % sha))