def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ p = PackData(path) try: entries = p.sorted_entries() basename = os.path.join( self.pack_dir, "pack-%s" % iter_sha1(entry[0] for entry in entries)) f = GitFile(basename + ".idx", "wb") try: write_pack_index_v2(f, entries, p.get_stored_checksum()) finally: f.close() finally: p.close() os.rename(path, basename + ".pack") final_pack = Pack(basename) self._add_known_pack(final_pack) return final_pack
def _load_packs(self): pack_files = [] try: self._pack_cache_time = os.stat(self.pack_dir).st_mtime pack_dir_contents = os.listdir(self.pack_dir) for name in pack_dir_contents: # TODO: verify that idx exists first if name.startswith("pack-") and name.endswith(".pack"): filename = os.path.join(self.pack_dir, name) pack_files.append((os.stat(filename).st_mtime, filename)) except OSError as e: if e.errno == errno.ENOENT: return [] raise pack_files.sort(reverse=True) suffix_len = len(".pack") result = [] try: for _, f in pack_files: result.append(Pack(f[:-suffix_len])) except: for p in result: p.close() raise return result
def move_in_thin_pack(self, f): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ f.seek(0) p = Pack('', resolve_ext_ref=self.get_raw) p._data = PackData.from_file(f, len(f.getvalue())) p._data.pack = p p._idx_load = lambda: MemoryPackIndex(p.data.sorted_entries(), p.data.get_stored_checksum()) pack_sha = p.index.objects_sha1() datafile = self.pack_transport.open_write_stream( "pack-%s.pack" % pack_sha.decode('ascii')) try: entries, data_sum = write_pack_objects(datafile, p.pack_tuples()) finally: datafile.close() entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()]) idxfile = self.pack_transport.open_write_stream( "pack-%s.idx" % pack_sha.decode('ascii')) try: write_pack_index_v2(idxfile, entries, data_sum) finally: idxfile.close() # TODO(jelmer): Just add new pack to the cache self._flush_pack_cache()
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. Note: The file should be on the same file system as the packs directory. Args: path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) index_name = basename + ".idx" if not os.path.exists(index_name): with GitFile(index_name, "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) for pack in self.packs: if pack._basename == basename: return pack target_pack = basename + ".pack" if sys.platform == "win32": # Windows might have the target pack file lingering. Attempt # removal, silently passing if the target does not exist. try: os.remove(target_pack) except FileNotFoundError: pass os.rename(path, target_pack) final_pack = Pack(basename) self._add_cached_pack(basename, final_pack) return final_pack
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + ".idx", "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) if self._pack_cache is None or self._pack_cache_stale(): self._update_pack_cache() try: return self._pack_cache[basename] except KeyError: pass else: os.unlink(path) os.rename(path, basename + ".pack") final_pack = Pack(basename) self._add_known_pack(basename, final_pack) return final_pack
def _update_pack_cache(self): try: pack_dir_contents = os.listdir(self.pack_dir) except OSError as e: if e.errno == errno.ENOENT: self._pack_cache_time = 0 self.close() return raise self._pack_cache_time = max( os.stat(self.pack_dir).st_mtime, time.time()) pack_files = set() for name in pack_dir_contents: if name.startswith("pack-") and name.endswith(".pack"): # verify that idx exists first (otherwise the pack was not yet # fully written) idx_name = os.path.splitext(name)[0] + ".idx" if idx_name in pack_dir_contents: pack_name = name[:-len(".pack")] pack_files.add(pack_name) # Open newly appeared pack files for f in pack_files: if f not in self._pack_cache: self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f)) # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close()
def _update_pack_cache(self): try: pack_dir_contents = os.listdir(self.pack_dir) except OSError as e: if e.errno == errno.ENOENT: self._pack_cache_time = 0 self.close() return raise self._pack_cache_time = os.stat(self.pack_dir).st_mtime pack_files = set() for name in pack_dir_contents: assert isinstance(name, basestring if sys.version_info[0] == 2 else str) # TODO: verify that idx exists first if name.startswith("pack-") and name.endswith(".pack"): pack_files.add(name[:-len(".pack")]) # Open newly appeared pack files for f in pack_files: if f not in self._pack_cache: self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f)) # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close()
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + ".idx", "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) if self._pack_cache is None or self._pack_cache_stale(): self._update_pack_cache() try: return self._pack_cache[basename] except KeyError: pass else: os.unlink(path) try: os.rename(path, basename + ".pack") except OSError as e: if e.errno == errno.EEXIST: # This can happen on Windows.. # It's safe to ignore this, since if the file already exists, # it should have the same contents as the one we just # generated. os.unlink(path) else: raise final_pack = Pack(basename) self._add_known_pack(basename, final_pack) return final_pack
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + ".idx", "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) for pack in self.packs: if pack._basename == basename: return pack target_pack = basename + '.pack' if sys.platform == 'win32': # Windows might have the target pack file lingering. Attempt # removal, silently passing if the target does not exist. try: os.remove(target_pack) except (IOError, OSError) as e: if e.errno != errno.ENOENT: raise os.rename(path, target_pack) final_pack = Pack(basename) self._add_cached_pack(basename, final_pack) return final_pack
def _update_pack_cache(self): """Read and iterate over new pack files and cache them.""" try: pack_dir_contents = os.listdir(self.pack_dir) except FileNotFoundError: self.close() return [] pack_files = set() for name in pack_dir_contents: if name.startswith("pack-") and name.endswith(".pack"): # verify that idx exists first (otherwise the pack was not yet # fully written) idx_name = os.path.splitext(name)[0] + ".idx" if idx_name in pack_dir_contents: pack_name = name[: -len(".pack")] pack_files.add(pack_name) # Open newly appeared pack files new_packs = [] for f in pack_files: if f not in self._pack_cache: pack = Pack(os.path.join(self.pack_dir, f)) new_packs.append(pack) self._pack_cache[f] = pack # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close() return new_packs
def _complete_thin_pack(self, f, path, copier, indexer): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param f: Open file object for the pack. :param path: Path to the pack file. :param copier: A PackStreamCopier to use for writing pack data. :param indexer: A PackIndexer for indexing the pack. """ entries = list(indexer) # Update the header with the new number of objects. f.seek(0) write_pack_header(f, len(entries) + len(indexer.ext_refs())) # Must flush before reading (http://bugs.python.org/issue3207) f.flush() # Rescan the rest of the pack, computing the SHA with the new header. new_sha = compute_file_sha(f, end_ofs=-20) # Must reposition before writing (http://bugs.python.org/issue3207) f.seek(0, os.SEEK_CUR) # Complete the pack. for ext_sha in indexer.ext_refs(): assert len(ext_sha) == 20 type_num, data = self.get_raw(ext_sha) offset = f.tell() crc32 = write_pack_object(f, type_num, data, sha=new_sha) entries.append((ext_sha, offset, crc32)) pack_sha = new_sha.digest() f.write(pack_sha) f.close() # Move the pack in. entries.sort() pack_base_name = self._get_pack_basepath(entries) try: os.rename(path, pack_base_name + '.pack') except WindowsError: os.remove(pack_base_name + '.pack') os.rename(path, pack_base_name + '.pack') # Write the index. index_file = GitFile(pack_base_name + '.idx', 'wb') try: write_pack_index_v2(index_file, entries, pack_sha) index_file.close() finally: index_file.abort() # Add the pack to the store and return it. final_pack = Pack(pack_base_name) final_pack.check_length_and_checksum() self._add_known_pack(pack_base_name, final_pack) return final_pack
def fetch_objects(self, determine_wants, graph_walker, progress=None): fd, path = tempfile.mkstemp(suffix=".pack") self.fetch_pack(determine_wants, graph_walker, lambda x: os.write(fd, x), progress) os.close(fd) basename = path[:-len(".pack")] p = PackData(path) p.create_index_v2(basename + ".idx") pack = Pack(basename) os.remove(path) return (len(p), pack.iterobjects())
def test_copy(self): origpack = self.get_pack(pack1_sha) self.assertEquals(True, origpack.index.check()) write_pack("Elch", [(x, "") for x in origpack.iterobjects()], len(origpack)) newpack = Pack("Elch") self.assertEquals(origpack, newpack) self.assertEquals(True, newpack.index.check()) self.assertEquals(origpack.name(), newpack.name()) self.assertEquals(origpack.index.get_pack_checksum(), newpack.index.get_pack_checksum()) self.assertTrue((origpack.index.version != newpack.index.version) or (origpack.index.get_stored_checksum() == newpack.index.get_stored_checksum()))
def move_in_pack(self, path): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ with PackData(path) as p: entries = p.sorted_entries() basename = self._get_pack_basepath(entries) with GitFile(basename + b'.idx', "wb") as f: write_pack_index_v2(f, entries, p.get_stored_checksum()) os.rename(path, basename + b'.pack') final_pack = Pack(basename) self._add_known_pack(basename, final_pack) return final_pack
def test_copy(self): with self.get_pack(pack1_sha) as origpack: self.assertSucceeds(origpack.index.check) basename = os.path.join(self.tempdir, 'Elch') write_pack(basename, origpack.pack_tuples()) with Pack(basename) as newpack: self.assertEqual(origpack, newpack) self.assertSucceeds(newpack.index.check) self.assertEqual(origpack.name(), newpack.name()) self.assertEqual(origpack.index.get_pack_checksum(), newpack.index.get_pack_checksum()) wrong_version = origpack.index.version != newpack.index.version orig_checksum = origpack.index.get_stored_checksum() new_checksum = newpack.index.get_stored_checksum() self.assertTrue(wrong_version or orig_checksum == new_checksum)
def run(self, args): opts, args = getopt(args, "", []) if args == []: print("Usage: dulwich dump-pack FILENAME") sys.exit(1) basename, _ = os.path.splitext(args[0]) x = Pack(basename) print("Object names checksum: %s" % x.name()) print("Checksum: %s" % sha_to_hex(x.get_stored_checksum())) if not x.check(): print("CHECKSUM DOES NOT MATCH") print("Length: %d" % len(x)) for name in x: try: print("\t%s" % x[name]) except KeyError as k: print("\t%s: Unable to resolve base %s" % (name, k)) except ApplyDeltaError as e: print("\t%s: Unable to apply delta: %r" % (name, e))
def commit(): from .fetch import import_git_objects os.fsync(fd) f.close() if os.path.getsize(path) == 0: return pd = PackData(path) pd.create_index_v2(path[:-5] + ".idx", self.object_store.get_raw) p = Pack(path[:-5]) with self.repository.lock_write(): self.repository.start_write_group() try: import_git_objects(self.repository, self.mapping, p.iterobjects(get_raw=self.get_raw), self.object_store) except BaseException: self.repository.abort_write_group() raise else: self.repository.commit_write_group()
def _update_pack_cache(self): try: pack_dir_contents = os.listdir(self.pack_dir) except OSError as e: if e.errno == errno.ENOENT: self._pack_cache_time = 0 self.close() return raise self._pack_cache_time = os.stat(self.pack_dir).st_mtime pack_files = set() for name in pack_dir_contents: assert type(name) is bytes # TODO: verify that idx exists first if name.startswith(b'pack-') and name.endswith(b'.pack'): pack_files.add(name[:-len(b'.pack')]) # Open newly appeared pack files for f in pack_files: if f not in self._pack_cache: self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f)) # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close()
def _complete_thin_pack(self, f, path, copier, indexer): """Move a specific file containing a pack into the pack directory. Note: The file should be on the same file system as the packs directory. Args: f: Open file object for the pack. path: Path to the pack file. copier: A PackStreamCopier to use for writing pack data. indexer: A PackIndexer for indexing the pack. """ entries = list(indexer) # Update the header with the new number of objects. f.seek(0) write_pack_header(f, len(entries) + len(indexer.ext_refs())) # Must flush before reading (http://bugs.python.org/issue3207) f.flush() # Rescan the rest of the pack, computing the SHA with the new header. new_sha = compute_file_sha(f, end_ofs=-20) # Must reposition before writing (http://bugs.python.org/issue3207) f.seek(0, os.SEEK_CUR) # Complete the pack. for ext_sha in indexer.ext_refs(): assert len(ext_sha) == 20 type_num, data = self.get_raw(ext_sha) offset = f.tell() crc32 = write_pack_object( f, type_num, data, sha=new_sha, compression_level=self.pack_compression_level, ) entries.append((ext_sha, offset, crc32)) pack_sha = new_sha.digest() f.write(pack_sha) f.close() # Move the pack in. entries.sort() pack_base_name = self._get_pack_basepath(entries) target_pack = pack_base_name + ".pack" if sys.platform == "win32": # Windows might have the target pack file lingering. Attempt # removal, silently passing if the target does not exist. try: os.remove(target_pack) except FileNotFoundError: pass os.rename(path, target_pack) # Write the index. index_file = GitFile(pack_base_name + ".idx", "wb") try: write_pack_index_v2(index_file, entries, pack_sha) index_file.close() finally: index_file.abort() # Add the pack to the store and return it. final_pack = Pack(pack_base_name) final_pack.check_length_and_checksum() self._add_cached_pack(pack_base_name, final_pack) return final_pack
pack_files = [] try: self._pack_cache_time = os.stat(self.pack_dir).st_mtime pack_dir_contents = os.listdir(self.pack_dir) for name in pack_dir_contents: # TODO: verify that idx exists first if name.startswith("pack-") and name.endswith(".pack"): filename = os.path.join(self.pack_dir, name) pack_files.append((os.stat(filename).st_mtime, filename)) except OSError, e: if e.errno == errno.ENOENT: return [] raise pack_files.sort(reverse=True) suffix_len = len(".pack") return [Pack(f[:-suffix_len]) for _, f in pack_files] def _pack_cache_stale(self): try: return os.stat(self.pack_dir).st_mtime > self._pack_cache_time except OSError, e: if e.errno == errno.ENOENT: return True raise def _get_shafile_path(self, sha): # Check from object dir return hex_to_filename(self.path, sha) def _iter_loose_objects(self): for base in os.listdir(self.path):
def get_objects(): pack = Pack(path[:-5]) for obj in pack.iterobjects(): yield obj
def _load_packs(self): if not os.path.exists(self.pack_dir): return for name in os.listdir(self.pack_dir): if name.startswith("pack-") and name.endswith(".pack"): yield Pack(os.path.join(self.pack_dir, name[:-len(".pack")]))
def test_copy(self): p = self.get_pack(pack1_sha) write_pack("Elch", p.iterobjects(), len(p)) self.assertEquals(p, Pack("Elch"))
def _copy_pack(self, origpack): basename = os.path.join(self.tempdir, 'somepack') write_pack(basename, origpack.pack_tuples()) return Pack(basename)
def make_pack(self, resolve_ext_ref): return Pack( self.pack_prefix, resolve_ext_ref=self.store.get_raw if resolve_ext_ref else None)
def get_pack(self, sha): return Pack(os.path.join(self.datadir, 'pack-%s' % sha))
def _add_known_pack(self, path): """Add a newly appeared pack to the cache by path. """ if self._pack_cache is not None: self._pack_cache.append(Pack(path))
def get_pack(self, sha): return Pack( os.path.join(self.datadir, 'pack-%s' % sha.decode('ascii')))
def get_pack(self, sha): return Pack(os.path.join(self.datadir, b'pack-' + sha))
def get_pack(self, sha): return Pack(os.path.join(self.datadir, "pack-%s" % sha.decode("ascii")))