def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
p = PackData(path)
try:
entries = p.sorted_entries()
basename = os.path.join(
self.pack_dir,
"pack-%s" % iter_sha1(entry[0] for entry in entries))
f = GitFile(basename + ".idx", "wb")
try:
write_pack_index_v2(f, entries, p.get_stored_checksum())
finally:
f.close()
finally:
p.close()
os.rename(path, basename + ".pack")
final_pack = Pack(basename)
self._add_known_pack(final_pack)
return final_pack
def _load_packs(self):
pack_files = []
try:
self._pack_cache_time = os.stat(self.pack_dir).st_mtime
pack_dir_contents = os.listdir(self.pack_dir)
for name in pack_dir_contents:
# TODO: verify that idx exists first
if name.startswith("pack-") and name.endswith(".pack"):
filename = os.path.join(self.pack_dir, name)
pack_files.append((os.stat(filename).st_mtime, filename))
except OSError as e:
if e.errno == errno.ENOENT:
return []
raise
pack_files.sort(reverse=True)
suffix_len = len(".pack")
result = []
try:
for _, f in pack_files:
result.append(Pack(f[:-suffix_len]))
except:
for p in result:
p.close()
raise
return result
def move_in_thin_pack(self, f):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
f.seek(0)
p = Pack('', resolve_ext_ref=self.get_raw)
p._data = PackData.from_file(f, len(f.getvalue()))
p._data.pack = p
p._idx_load = lambda: MemoryPackIndex(p.data.sorted_entries(),
p.data.get_stored_checksum())
pack_sha = p.index.objects_sha1()
datafile = self.pack_transport.open_write_stream(
"pack-%s.pack" % pack_sha.decode('ascii'))
try:
entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
finally:
datafile.close()
entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
idxfile = self.pack_transport.open_write_stream(
"pack-%s.idx" % pack_sha.decode('ascii'))
try:
write_pack_index_v2(idxfile, entries, data_sum)
finally:
idxfile.close()
# TODO(jelmer): Just add new pack to the cache
self._flush_pack_cache()
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
Note: The file should be on the same file system as the
packs directory.
Args:
path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
index_name = basename + ".idx"
if not os.path.exists(index_name):
with GitFile(index_name, "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
for pack in self.packs:
if pack._basename == basename:
return pack
target_pack = basename + ".pack"
if sys.platform == "win32":
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except FileNotFoundError:
pass
os.rename(path, target_pack)
final_pack = Pack(basename)
self._add_cached_pack(basename, final_pack)
return final_pack
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + ".idx", "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
if self._pack_cache is None or self._pack_cache_stale():
self._update_pack_cache()
try:
return self._pack_cache[basename]
except KeyError:
pass
else:
os.unlink(path)
os.rename(path, basename + ".pack")
final_pack = Pack(basename)
self._add_known_pack(basename, final_pack)
return final_pack
def _update_pack_cache(self):
try:
pack_dir_contents = os.listdir(self.pack_dir)
except OSError as e:
if e.errno == errno.ENOENT:
self._pack_cache_time = 0
self.close()
return
raise
self._pack_cache_time = max(
os.stat(self.pack_dir).st_mtime, time.time())
pack_files = set()
for name in pack_dir_contents:
if name.startswith("pack-") and name.endswith(".pack"):
# verify that idx exists first (otherwise the pack was not yet
# fully written)
idx_name = os.path.splitext(name)[0] + ".idx"
if idx_name in pack_dir_contents:
pack_name = name[:-len(".pack")]
pack_files.add(pack_name)
# Open newly appeared pack files
for f in pack_files:
if f not in self._pack_cache:
self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f))
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
def _update_pack_cache(self):
try:
pack_dir_contents = os.listdir(self.pack_dir)
except OSError as e:
if e.errno == errno.ENOENT:
self._pack_cache_time = 0
self.close()
return
raise
self._pack_cache_time = os.stat(self.pack_dir).st_mtime
pack_files = set()
for name in pack_dir_contents:
assert isinstance(name,
basestring if sys.version_info[0] == 2 else str)
# TODO: verify that idx exists first
if name.startswith("pack-") and name.endswith(".pack"):
pack_files.add(name[:-len(".pack")])
# Open newly appeared pack files
for f in pack_files:
if f not in self._pack_cache:
self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f))
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + ".idx", "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
if self._pack_cache is None or self._pack_cache_stale():
self._update_pack_cache()
try:
return self._pack_cache[basename]
except KeyError:
pass
else:
os.unlink(path)
try:
os.rename(path, basename + ".pack")
except OSError as e:
if e.errno == errno.EEXIST:
# This can happen on Windows..
# It's safe to ignore this, since if the file already exists,
# it should have the same contents as the one we just
# generated.
os.unlink(path)
else:
raise
final_pack = Pack(basename)
self._add_known_pack(basename, final_pack)
return final_pack
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + ".idx", "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
for pack in self.packs:
if pack._basename == basename:
return pack
target_pack = basename + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
final_pack = Pack(basename)
self._add_cached_pack(basename, final_pack)
return final_pack
def _update_pack_cache(self):
"""Read and iterate over new pack files and cache them."""
try:
pack_dir_contents = os.listdir(self.pack_dir)
except FileNotFoundError:
self.close()
return []
pack_files = set()
for name in pack_dir_contents:
if name.startswith("pack-") and name.endswith(".pack"):
# verify that idx exists first (otherwise the pack was not yet
# fully written)
idx_name = os.path.splitext(name)[0] + ".idx"
if idx_name in pack_dir_contents:
pack_name = name[: -len(".pack")]
pack_files.add(pack_name)
# Open newly appeared pack files
new_packs = []
for f in pack_files:
if f not in self._pack_cache:
pack = Pack(os.path.join(self.pack_dir, f))
new_packs.append(pack)
self._pack_cache[f] = pack
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
return new_packs
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
try:
os.rename(path, pack_base_name + '.pack')
except WindowsError:
os.remove(pack_base_name + '.pack')
os.rename(path, pack_base_name + '.pack')
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(pack_base_name, final_pack)
return final_pack
def fetch_objects(self, determine_wants, graph_walker, progress=None):
fd, path = tempfile.mkstemp(suffix=".pack")
self.fetch_pack(determine_wants, graph_walker,
lambda x: os.write(fd, x), progress)
os.close(fd)
basename = path[:-len(".pack")]
p = PackData(path)
p.create_index_v2(basename + ".idx")
pack = Pack(basename)
os.remove(path)
return (len(p), pack.iterobjects())
def test_copy(self):
origpack = self.get_pack(pack1_sha)
self.assertEquals(True, origpack.index.check())
write_pack("Elch", [(x, "") for x in origpack.iterobjects()],
len(origpack))
newpack = Pack("Elch")
self.assertEquals(origpack, newpack)
self.assertEquals(True, newpack.index.check())
self.assertEquals(origpack.name(), newpack.name())
self.assertEquals(origpack.index.get_pack_checksum(),
newpack.index.get_pack_checksum())
self.assertTrue((origpack.index.version != newpack.index.version)
or (origpack.index.get_stored_checksum()
== newpack.index.get_stored_checksum()))
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + b'.idx', "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
os.rename(path, basename + b'.pack')
final_pack = Pack(basename)
self._add_known_pack(basename, final_pack)
return final_pack
def test_copy(self):
with self.get_pack(pack1_sha) as origpack:
self.assertSucceeds(origpack.index.check)
basename = os.path.join(self.tempdir, 'Elch')
write_pack(basename, origpack.pack_tuples())
with Pack(basename) as newpack:
self.assertEqual(origpack, newpack)
self.assertSucceeds(newpack.index.check)
self.assertEqual(origpack.name(), newpack.name())
self.assertEqual(origpack.index.get_pack_checksum(),
newpack.index.get_pack_checksum())
wrong_version = origpack.index.version != newpack.index.version
orig_checksum = origpack.index.get_stored_checksum()
new_checksum = newpack.index.get_stored_checksum()
self.assertTrue(wrong_version or orig_checksum == new_checksum)
def run(self, args):
opts, args = getopt(args, "", [])
if args == []:
print("Usage: dulwich dump-pack FILENAME")
sys.exit(1)
basename, _ = os.path.splitext(args[0])
x = Pack(basename)
print("Object names checksum: %s" % x.name())
print("Checksum: %s" % sha_to_hex(x.get_stored_checksum()))
if not x.check():
print("CHECKSUM DOES NOT MATCH")
print("Length: %d" % len(x))
for name in x:
try:
print("\t%s" % x[name])
except KeyError as k:
print("\t%s: Unable to resolve base %s" % (name, k))
except ApplyDeltaError as e:
print("\t%s: Unable to apply delta: %r" % (name, e))
def commit():
from .fetch import import_git_objects
os.fsync(fd)
f.close()
if os.path.getsize(path) == 0:
return
pd = PackData(path)
pd.create_index_v2(path[:-5] + ".idx", self.object_store.get_raw)
p = Pack(path[:-5])
with self.repository.lock_write():
self.repository.start_write_group()
try:
import_git_objects(self.repository, self.mapping,
p.iterobjects(get_raw=self.get_raw),
self.object_store)
except BaseException:
self.repository.abort_write_group()
raise
else:
self.repository.commit_write_group()
def _update_pack_cache(self):
try:
pack_dir_contents = os.listdir(self.pack_dir)
except OSError as e:
if e.errno == errno.ENOENT:
self._pack_cache_time = 0
self.close()
return
raise
self._pack_cache_time = os.stat(self.pack_dir).st_mtime
pack_files = set()
for name in pack_dir_contents:
assert type(name) is bytes
# TODO: verify that idx exists first
if name.startswith(b'pack-') and name.endswith(b'.pack'):
pack_files.add(name[:-len(b'.pack')])
# Open newly appeared pack files
for f in pack_files:
if f not in self._pack_cache:
self._pack_cache[f] = Pack(os.path.join(self.pack_dir, f))
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
Note: The file should be on the same file system as the
packs directory.
Args:
f: Open file object for the pack.
path: Path to the pack file.
copier: A PackStreamCopier to use for writing pack data.
indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(
f,
type_num,
data,
sha=new_sha,
compression_level=self.pack_compression_level,
)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + ".pack"
if sys.platform == "win32":
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except FileNotFoundError:
pass
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + ".idx", "wb")
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
pack_files = []
try:
self._pack_cache_time = os.stat(self.pack_dir).st_mtime
pack_dir_contents = os.listdir(self.pack_dir)
for name in pack_dir_contents:
# TODO: verify that idx exists first
if name.startswith("pack-") and name.endswith(".pack"):
filename = os.path.join(self.pack_dir, name)
pack_files.append((os.stat(filename).st_mtime, filename))
except OSError, e:
if e.errno == errno.ENOENT:
return []
raise
pack_files.sort(reverse=True)
suffix_len = len(".pack")
return [Pack(f[:-suffix_len]) for _, f in pack_files]
def _pack_cache_stale(self):
try:
return os.stat(self.pack_dir).st_mtime > self._pack_cache_time
except OSError, e:
if e.errno == errno.ENOENT:
return True
raise
def _get_shafile_path(self, sha):
# Check from object dir
return hex_to_filename(self.path, sha)
def _iter_loose_objects(self):
for base in os.listdir(self.path):
def get_objects():
pack = Pack(path[:-5])
for obj in pack.iterobjects():
yield obj
def _load_packs(self):
if not os.path.exists(self.pack_dir):
return
for name in os.listdir(self.pack_dir):
if name.startswith("pack-") and name.endswith(".pack"):
yield Pack(os.path.join(self.pack_dir, name[:-len(".pack")]))
def test_copy(self):
p = self.get_pack(pack1_sha)
write_pack("Elch", p.iterobjects(), len(p))
self.assertEquals(p, Pack("Elch"))
def _copy_pack(self, origpack):
basename = os.path.join(self.tempdir, 'somepack')
write_pack(basename, origpack.pack_tuples())
return Pack(basename)
def make_pack(self, resolve_ext_ref):
return Pack(
self.pack_prefix,
resolve_ext_ref=self.store.get_raw if resolve_ext_ref else None)
def get_pack(self, sha):
return Pack(os.path.join(self.datadir, 'pack-%s' % sha))
def _add_known_pack(self, path):
"""Add a newly appeared pack to the cache by path.
"""
if self._pack_cache is not None:
self._pack_cache.append(Pack(path))
def get_pack(self, sha):
return Pack(
os.path.join(self.datadir, 'pack-%s' % sha.decode('ascii')))
def get_pack(self, sha):
return Pack(os.path.join(self.datadir, b'pack-' + sha))
def get_pack(self, sha):
return Pack(os.path.join(self.datadir,
"pack-%s" % sha.decode("ascii")))
