def test_send_pack_new_ref_only(self):
self.rin.write(
b'0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/master\x00report-status delete-refs ofs-delta\n'
b'0000000eunpack ok\n'
b'0019ok refs/heads/blah12\n'
b'0000')
self.rin.seek(0)
def determine_wants(refs):
return {
b'refs/heads/blah12':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c',
b'refs/heads/master':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_contents(have, want):
return {}
f = BytesIO()
write_pack_objects(f, {})
self.client.send_pack('/', determine_wants, generate_pack_contents)
self.assertIn(
self.rout.getvalue(),
[b'007f0000000000000000000000000000000000000000 '
b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/blah12\x00report-status ofs-delta0000' +
f.getvalue(),
b'007f0000000000000000000000000000000000000000 '
b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/blah12\x00ofs-delta report-status0000' +
f.getvalue()])
def test_send_pack_new_ref_only(self):
self.rin.write(
b'0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/master\x00report-status delete-refs ofs-delta\n'
b'0000000eunpack ok\n'
b'0019ok refs/heads/blah12\n'
b'0000')
self.rin.seek(0)
def determine_wants(refs):
return {
b'refs/heads/blah12':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c',
b'refs/heads/master':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_data(have, want, ofs_delta=False):
return 0, []
f = BytesIO()
write_pack_objects(f, {})
self.client.send_pack('/', determine_wants, generate_pack_data)
self.assertIn(
self.rout.getvalue(),
[b'007f0000000000000000000000000000000000000000 '
b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/blah12\x00report-status ofs-delta0000' +
f.getvalue(),
b'007f0000000000000000000000000000000000000000 '
b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/blah12\x00ofs-delta report-status0000' +
f.getvalue()])
def test_send_pack_new_ref_only(self):
self.rin.write(
'0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
'refs/heads/master\x00report-status delete-refs ofs-delta\n'
'0000000eunpack ok\n'
'0019ok refs/heads/blah12\n'
'0000')
self.rin.seek(0)
def determine_wants(refs):
return {
'refs/heads/blah12':
'310ca9477129b8586fa2afc779c1f57cf64bba6c',
'refs/heads/master': '310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_contents(have, want):
return {}
f = BytesIO()
write_pack_objects(f, {})
self.client.send_pack('/', determine_wants, generate_pack_contents)
self.assertIn(
self.rout.getvalue(),
['007f0000000000000000000000000000000000000000 '
'310ca9477129b8586fa2afc779c1f57cf64bba6c '
'refs/heads/blah12\x00report-status ofs-delta0000%s'
% f.getvalue(),
'007f0000000000000000000000000000000000000000 '
'310ca9477129b8586fa2afc779c1f57cf64bba6c '
'refs/heads/blah12\x00ofs-delta report-status0000%s'
% f.getvalue()])
def handle(self):
write = lambda x: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x)
graph_walker = ProtocolGraphWalker(self, self.repo.object_store,
self.repo.get_peeled)
objects_iter = self.repo.fetch_objects(
graph_walker.determine_wants, graph_walker, self.progress,
get_tagged=self.get_tagged)
# Note the fact that client is only processing responses related
# to the have lines it sent, and any other data (including side-
# band) will be be considered a fatal error.
self._processing_have_lines = True
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
# Also, if it also happens that the object_iter is instantiated
# with a graph walker with an implementation that talks over the
# wire (which is this instance of this class) this will actually
# iterate through everything and write things out to the wire.
if len(objects_iter) == 0:
return
# The provided haves are processed, and it is safe to send side-
# band data now.
self._processing_have_lines = False
self.progress(b"dul-daemon says what\n")
self.progress(("counting objects: %d, done.\n" % len(objects_iter)).encode('ascii'))
write_pack_objects(ProtocolFile(None, write), objects_iter)
self.progress(b"how was that, then?\n")
# we are done
self.proto.write_pkt_line(None)
def test_send_pack_new_ref_only(self):
self.rin.write(
b'0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/master\x00report-status delete-refs ofs-delta\n'
b'0000000eunpack ok\n'
b'0019ok refs/heads/blah12\n'
b'0000')
self.rin.seek(0)
def update_refs(refs):
return {
b'refs/heads/blah12':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c',
b'refs/heads/master':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_data(have, want, ofs_delta=False):
return 0, []
f = BytesIO()
write_pack_objects(f, {})
self.client.send_pack('/', update_refs, generate_pack_data)
self.assertEqual(
self.rout.getvalue(),
b'008b0000000000000000000000000000000000000000 '
b'310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/blah12\x00delete-refs ofs-delta report-status0000' +
f.getvalue())
def handle(self):
write = lambda x: self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x)
graph_walker = ProtocolGraphWalker(self, self.repo.object_store,
self.repo.get_peeled)
objects_iter = self.repo.fetch_objects(graph_walker.determine_wants,
graph_walker,
self.progress,
get_tagged=self.get_tagged)
# Note the fact that client is only processing responses related
# to the have lines it sent, and any other data (including side-
# band) will be be considered a fatal error.
self._processing_have_lines = True
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
# Also, if it also happens that the object_iter is instantiated
# with a graph walker with an implementation that talks over the
# wire (which is this instance of this class) this will actually
# iterate through everything and write things out to the wire.
if len(objects_iter) == 0:
return
# The provided haves are processed, and it is safe to send side-
# band data now.
self._processing_have_lines = False
self.progress(b"dul-daemon says what\n")
self.progress(("counting objects: %d, done.\n" %
len(objects_iter)).encode('ascii'))
write_pack_objects(ProtocolFile(None, write), objects_iter)
self.progress(b"how was that, then?\n")
# we are done
self.proto.write_pkt_line(None)
def test_add_pack(self):
o = MemoryObjectStore()
f, commit, abort = o.add_pack()
try:
b = make_object(Blob, data="more yummy data")
write_pack_objects(f, [(b, None)])
except:
abort()
raise
else:
commit()
def test_add_pack(self):
o = DiskObjectStore(self.store_dir)
f, commit, abort = o.add_pack()
try:
b = make_object(Blob, data=b"more yummy data")
write_pack_objects(f, [(b, None)])
except BaseException:
abort()
raise
else:
commit()
def test_add_pack(self):
o = MemoryObjectStore()
f, commit, abort = o.add_pack()
try:
b = make_object(Blob, data=b"more yummy data")
write_pack_objects(f, [(b, None)])
except BaseException:
abort()
raise
else:
commit()
def test_add_pack(self):
o = DiskObjectStore(self.store_dir)
f, commit, abort = o.add_pack()
try:
b = make_object(Blob, data=b"more yummy data")
write_pack_objects(f, [(b, None)])
except:
abort()
raise
else:
commit()
def add_objects(self, objects):
"""Add a set of objects to this object store.
:param objects: Iterable over objects, should support __len__.
:return: Pack object of the objects written.
"""
if len(objects) == 0:
# Don't bother writing an empty pack file
return
f, commit = self.add_pack()
write_pack_objects(f, objects)
return commit()
def add_objects(self, objects):
"""Add a set of objects to this object store.
:param objects: Iterable over objects, should support __len__.
:return: Pack object of the objects written.
"""
if len(objects) == 0:
# Don't bother writing an empty pack file
return
f, commit = self.add_pack()
write_pack_objects(f, objects)
return commit()
def handle(self):
def write(x):
return self.proto.write_sideband(SIDE_BAND_CHANNEL_DATA, x)
graph_walker = _ProtocolGraphWalker(
self,
self.repo.object_store,
self.repo.get_peeled,
self.repo.refs.get_symrefs,
)
wants = []
def wants_wrapper(refs, **kwargs):
wants.extend(graph_walker.determine_wants(refs, **kwargs))
return wants
objects_iter = self.repo.fetch_objects(
wants_wrapper,
graph_walker,
self.progress,
get_tagged=self.get_tagged,
)
# Note the fact that client is only processing responses related
# to the have lines it sent, and any other data (including side-
# band) will be be considered a fatal error.
self._processing_have_lines = True
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
# Also, if it also happens that the object_iter is instantiated
# with a graph walker with an implementation that talks over the
# wire (which is this instance of this class) this will actually
# iterate through everything and write things out to the wire.
if len(wants) == 0:
return
# The provided haves are processed, and it is safe to send side-
# band data now.
self._processing_have_lines = False
if not graph_walker.handle_done(
not self.has_capability(CAPABILITY_NO_DONE), self._done_received
):
return
self.progress(
("counting objects: %d, done.\n" % len(objects_iter)).encode("ascii")
)
write_pack_objects(ProtocolFile(None, write), objects_iter)
# we are done
self.proto.write_pkt_line(None)
def send_pack(self, path, determine_wants, generate_pack_contents,
progress=None):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of the
shas of the objects to upload.
:param progress: Optional callback called with progress updates
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
proto, unused_can_read = self._connect('receive-pack', path)
old_refs, server_capabilities = self._read_refs(proto)
negotiated_capabilities = self._send_capabilities & server_capabilities
try:
new_refs = determine_wants(dict(old_refs))
except:
proto.write_pkt_line(None)
raise
if new_refs is None:
proto.write_pkt_line(None)
return old_refs
(have, want) = self._handle_receive_pack_head(proto,
negotiated_capabilities, old_refs, new_refs)
if not want and old_refs == new_refs:
return new_refs
objects = generate_pack_contents(have, want)
if len(objects) > 0:
entries, sha = write_pack_objects(proto.write_file(), objects)
self._handle_receive_pack_tail(proto, negotiated_capabilities,
progress)
return new_refs
def move_in_thin_pack(self, f):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
f.seek(0)
p = Pack('', resolve_ext_ref=self.get_raw)
p._data = PackData.from_file(f, len(f.getvalue()))
p._data.pack = p
p._idx_load = lambda: MemoryPackIndex(p.data.sorted_entries(),
p.data.get_stored_checksum())
pack_sha = p.index.objects_sha1()
datafile = self.pack_transport.open_write_stream(
"pack-%s.pack" % pack_sha.decode('ascii'))
try:
entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
finally:
datafile.close()
entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
idxfile = self.pack_transport.open_write_stream(
"pack-%s.idx" % pack_sha.decode('ascii'))
try:
write_pack_index_v2(idxfile, entries, data_sum)
finally:
idxfile.close()
# TODO(jelmer): Just add new pack to the cache
self._flush_pack_cache()
def send_pack(self, path, determine_wants, generate_pack_contents, progress=None):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of the
shas of the objects to upload.
:param progress: Optional callback called with progress updates
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
proto, unused_can_read = self._connect("receive-pack", path)
old_refs, server_capabilities = self._read_refs(proto)
negotiated_capabilities = self._send_capabilities & server_capabilities
try:
new_refs = determine_wants(dict(old_refs))
except:
proto.write_pkt_line(None)
raise
if new_refs is None:
proto.write_pkt_line(None)
return old_refs
(have, want) = self._handle_receive_pack_head(proto, negotiated_capabilities, old_refs, new_refs)
if not want and old_refs == new_refs:
return new_refs
objects = generate_pack_contents(have, want)
if len(objects) > 0:
entries, sha = write_pack_objects(proto.write_file(), objects)
self._handle_receive_pack_tail(proto, negotiated_capabilities, progress)
return new_refs
def send_pack(self, path, determine_wants, generate_pack_contents, progress=None):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of the
shas of the objects to upload.
:param progress: Optional progress function
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
url = self._get_url(path)
old_refs, server_capabilities = self._discover_references("git-receive-pack", url)
negotiated_capabilities = self._send_capabilities & server_capabilities
new_refs = determine_wants(dict(old_refs))
if new_refs is None:
return old_refs
if self.dumb:
raise NotImplementedError(self.fetch_pack)
req_data = StringIO()
req_proto = Protocol(None, req_data.write)
(have, want) = self._handle_receive_pack_head(req_proto, negotiated_capabilities, old_refs, new_refs)
if not want and old_refs == new_refs:
return new_refs
objects = generate_pack_contents(have, want)
if len(objects) > 0:
entries, sha = write_pack_objects(req_proto.write_file(), objects)
resp = self._smart_request("git-receive-pack", url, data=req_data.getvalue())
resp_proto = Protocol(resp.read, None)
self._handle_receive_pack_tail(resp_proto, negotiated_capabilities, progress)
return new_refs
def test_send_pack_new_ref_only(self):
self.rin.write(
'0063310ca9477129b8586fa2afc779c1f57cf64bba6c '
'refs/heads/master\x00report-status delete-refs ofs-delta\n'
'0000000eunpack ok\n'
'0019ok refs/heads/blah12\n'
'0000')
self.rin.seek(0)
def determine_wants(refs):
return {
'refs/heads/blah12':
'310ca9477129b8586fa2afc779c1f57cf64bba6c',
'refs/heads/master': '310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_contents(have, want):
return {}
f = StringIO()
empty_pack = write_pack_objects(f, {})
self.client.send_pack('/', determine_wants, generate_pack_contents)
self.assertEqual(
self.rout.getvalue(),
'007f0000000000000000000000000000000000000000 '
'310ca9477129b8586fa2afc779c1f57cf64bba6c '
'refs/heads/blah12\x00report-status ofs-delta0000%s' %
f.getvalue())
def handle(self):
write = lambda x: self.proto.write_sideband(1, x)
graph_walker = ProtocolGraphWalker(self, self.repo.object_store, self.repo.get_peeled)
objects_iter = self.repo.fetch_objects(
graph_walker.determine_wants, graph_walker, self.progress, get_tagged=self.get_tagged
)
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
if objects_iter is None:
return
self.progress("counting objects: %d, done.\n" % len(objects_iter))
write_pack_objects(ProtocolFile(None, write), objects_iter)
# we are done
self.proto.write("0000")
def add_objects(self, objects):
"""Add a set of objects to this object store.
:param objects: Iterable over (object, path) tuples, should support
__len__.
:return: Pack object of the objects written.
"""
if len(objects) == 0:
# Don't bother writing an empty pack file
return
f, commit, abort = self.add_pack()
try:
write_pack_objects(f, objects)
except:
abort()
raise
else:
return commit()
def add_objects(self, objects):
"""Add a set of objects to this object store.
:param objects: Iterable over (object, path) tuples, should support
__len__.
:return: Pack object of the objects written.
"""
if len(objects) == 0:
# Don't bother writing an empty pack file
return
f, commit, abort = self.add_pack()
try:
write_pack_objects(f, objects)
except:
abort()
raise
else:
return commit()
def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
progress=None):
"""Retrieve a pack from a git smart server.
:param determine_wants: Callback that returns list of commits to fetch
:param graph_walker: Object with next() and ack().
:param pack_data: Callback called for each bit of data in the pack
:param progress: Callback for progress reports (strings)
"""
from dulwich.repo import Repo
with closing(Repo(path)) as r:
objects_iter = r.fetch_objects(determine_wants, graph_walker, progress)
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
if objects_iter is None:
return
write_pack_objects(ProtocolFile(None, pack_data), objects_iter)
def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
progress=None):
"""Retrieve a pack from a git smart server.
:param determine_wants: Callback that returns list of commits to fetch
:param graph_walker: Object with next() and ack().
:param pack_data: Callback called for each bit of data in the pack
:param progress: Callback for progress reports (strings)
"""
from dulwich.repo import Repo
with closing(Repo(path)) as r:
objects_iter = r.fetch_objects(determine_wants, graph_walker, progress)
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
if objects_iter is None:
return
write_pack_objects(ProtocolFile(None, pack_data), objects_iter)
def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
progress=None):
"""Retrieve a pack from a git smart server.
:param determine_wants: Callback that returns list of commits to fetch
:param graph_walker: Object with next() and ack().
:param pack_data: Callback called for each bit of data in the pack
:param progress: Callback for progress reports (strings)
:return: Dictionary with all remote refs (not just those fetched)
"""
with self._open_repo(path) as r:
objects_iter = r.fetch_objects(determine_wants, graph_walker, progress)
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
if objects_iter is None:
return
write_pack_objects(ProtocolFile(None, pack_data), objects_iter)
return r.get_refs()
def test_send_pack_new_ref(self):
self.rin.write(
'0064310ca9477129b8586fa2afc779c1f57cf64bba6c '
'refs/heads/master\x00 report-status delete-refs ofs-delta\n'
'0000000eunpack ok\n'
'0019ok refs/heads/blah12\n'
'0000')
self.rin.seek(0)
tree = Tree()
commit = Commit()
commit.tree = tree
commit.parents = []
commit.author = commit.committer = 'test user'
commit.commit_time = commit.author_time = 1174773719
commit.commit_timezone = commit.author_timezone = 0
commit.encoding = 'UTF-8'
commit.message = 'test message'
def determine_wants(refs):
return {
'refs/heads/blah12': commit.id,
'refs/heads/master': '310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_contents(have, want):
return [
(commit, None),
(tree, ''),
]
f = BytesIO()
write_pack_objects(f, generate_pack_contents(None, None))
self.client.send_pack('/', determine_wants, generate_pack_contents)
self.assertIn(self.rout.getvalue(), [
'007f0000000000000000000000000000000000000000 %s '
'refs/heads/blah12\x00report-status ofs-delta0000%s' %
(commit.id, f.getvalue()),
'007f0000000000000000000000000000000000000000 %s '
'refs/heads/blah12\x00ofs-delta report-status0000%s' %
(commit.id, f.getvalue())
])
def handle(self):
write = lambda x: self.proto.write_sideband(1, x)
graph_walker = ProtocolGraphWalker(self, self.repo.object_store,
self.repo.get_peeled)
objects_iter = self.repo.fetch_objects(
graph_walker.determine_wants, graph_walker, self.progress,
get_tagged=self.get_tagged)
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
if len(objects_iter) == 0:
return
self.progress("dul-daemon says what\n")
self.progress("counting objects: %d, done.\n" % len(objects_iter))
write_pack_objects(ProtocolFile(None, write), objects_iter)
self.progress("how was that, then?\n")
# we are done
self.proto.write("0000")
def test_send_pack_new_ref(self):
self.rin.write(
'0064310ca9477129b8586fa2afc779c1f57cf64bba6c '
'refs/heads/master\x00 report-status delete-refs ofs-delta\n'
'0000000eunpack ok\n'
'0019ok refs/heads/blah12\n'
'0000')
self.rin.seek(0)
tree = Tree()
commit = Commit()
commit.tree = tree
commit.parents = []
commit.author = commit.committer = 'test user'
commit.commit_time = commit.author_time = 1174773719
commit.commit_timezone = commit.author_timezone = 0
commit.encoding = 'UTF-8'
commit.message = 'test message'
def determine_wants(refs):
return {
'refs/heads/blah12': commit.id,
'refs/heads/master': '310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_contents(have, want):
return [(commit, None), (tree, ''), ]
f = BytesIO()
write_pack_objects(f, generate_pack_contents(None, None))
self.client.send_pack('/', determine_wants, generate_pack_contents)
self.assertIn(
self.rout.getvalue(),
['007f0000000000000000000000000000000000000000 %s '
'refs/heads/blah12\x00report-status ofs-delta0000%s'
% (commit.id, f.getvalue()),
'007f0000000000000000000000000000000000000000 %s '
'refs/heads/blah12\x00ofs-delta report-status0000%s'
% (commit.id, f.getvalue())])
def send_pack(self, path, determine_wants, generate_pack_contents):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of the
shas of the objects to upload.
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
proto, unused_can_read = self._connect('receive-pack', path)
old_refs, server_capabilities = self._read_refs(proto)
if 'report-status' not in server_capabilities:
self._send_capabilities.remove('report-status')
new_refs = determine_wants(old_refs)
if not new_refs:
proto.write_pkt_line(None)
return {}
want = []
have = [x for x in old_refs.values() if not x == ZERO_SHA]
sent_capabilities = False
for refname in set(new_refs.keys() + old_refs.keys()):
old_sha1 = old_refs.get(refname, ZERO_SHA)
new_sha1 = new_refs.get(refname, ZERO_SHA)
if old_sha1 != new_sha1:
if sent_capabilities:
proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
refname))
else:
proto.write_pkt_line(
'%s %s %s\0%s' % (old_sha1, new_sha1, refname,
' '.join(self._send_capabilities)))
sent_capabilities = True
if new_sha1 not in have and new_sha1 != ZERO_SHA:
want.append(new_sha1)
proto.write_pkt_line(None)
if not want:
return new_refs
objects = generate_pack_contents(have, want)
entries, sha = write_pack_objects(proto.write_file(), objects)
if 'report-status' in self._send_capabilities:
self._parse_status_report(proto)
# wait for EOF before returning
data = proto.read()
if data:
raise SendPackError('Unexpected response %r' % data)
return new_refs
def send_pack(self,
path,
determine_wants,
generate_pack_contents,
progress=None):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of
the shas of the objects to upload.
:param progress: Optional progress function
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
url = self._get_url(path)
old_refs, server_capabilities = self._discover_references(
"git-receive-pack", url)
negotiated_capabilities = self._send_capabilities & server_capabilities
if 'report-status' in negotiated_capabilities:
self._report_status_parser = ReportStatusParser()
new_refs = determine_wants(dict(old_refs))
if new_refs is None:
return old_refs
if self.dumb:
raise NotImplementedError(self.fetch_pack)
req_data = BytesIO()
req_proto = Protocol(None, req_data.write)
(have, want) = self._handle_receive_pack_head(req_proto,
negotiated_capabilities,
old_refs, new_refs)
if not want and old_refs == new_refs:
return new_refs
objects = generate_pack_contents(have, want)
if len(objects) > 0:
entries, sha = write_pack_objects(req_proto.write_file(), objects)
resp = self._smart_request("git-receive-pack",
url,
data=req_data.getvalue())
try:
resp_proto = Protocol(resp.read, None)
self._handle_receive_pack_tail(resp_proto, negotiated_capabilities,
progress)
return new_refs
finally:
resp.close()
def pack_objects(repo, object_ids, packf, idxf, delta_window_size=None):
"""Pack objects into a file.
:param repo: Path to the repository
:param object_ids: List of object ids to write
:param packf: File-like object to write to
:param idxf: File-like object to write to (can be None)
"""
with open_repo_closing(repo) as r:
entries, data_sum = write_pack_objects(
packf,
r.object_store.iter_shas((oid, None) for oid in object_ids),
delta_window_size=delta_window_size)
if idxf is not None:
entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
write_pack_index(idxf, entries, data_sum)
def pack_objects(repo, object_ids, packf, idxf, delta_window_size=None):
"""Pack objects into a file.
:param repo: Path to the repository
:param object_ids: List of object ids to write
:param packf: File-like object to write to
:param idxf: File-like object to write to (can be None)
"""
with open_repo_closing(repo) as r:
entries, data_sum = write_pack_objects(
packf,
r.object_store.iter_shas((oid, None) for oid in object_ids),
delta_window_size=delta_window_size)
if idxf is not None:
entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
write_pack_index(idxf, entries, data_sum)
def test_add_pack(self):
o = DiskObjectStore(self.store_dir)
f, commit = o.add_pack()
b = make_object(Blob, data="more yummy data")
write_pack_objects(f, [(b, None)])
commit()
def resolve_obj(e):
if e.pack_type_num in pack.DELTA_TYPES:
res = p.resolve_object(offset=e.offset, type=e.pack_type_num, obj=(e.delta_base, e.decomp_chunks))
e.obj_type_num = res[0]
e.obj_chunks = res[1]
e.delta_base = None
return e
def map_object_for_write(e):
#print(e)
# Need to resolve object, just to calculate its SHA1
e = resolve_obj(e)
return e.sha_file()
p = pack.PackData(sys.argv[1])
f = open("out.pack", "wb")
obj_map, check_sha = pack.write_pack_objects(f, [(map_object_for_write(e), None) for e in p._iter_unpacked()])
f.close()
pack_sha = binascii.hexlify(check_sha).decode("ascii")
os.rename("out.pack", "out-%s.pack" % pack_sha)
entries = [(k, v[0], v[1]) for (k, v) in obj_map.items()]
entries.sort()
with open("out-%s.idx" % pack_sha, "wb") as f:
pack.write_pack_index_v2(f, entries, check_sha)
def send_pack(self, path, determine_wants, generate_pack_contents,
progress=None):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of the
shas of the objects to upload.
:param progress: Optional callback called with progress updates
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
proto, unused_can_read = self._connect('receive-pack', path)
old_refs, server_capabilities = read_pkt_refs(proto)
negotiated_capabilities = self._send_capabilities & server_capabilities
if 'report-status' in negotiated_capabilities:
self._report_status_parser = ReportStatusParser()
report_status_parser = self._report_status_parser
try:
new_refs = orig_new_refs = determine_wants(dict(old_refs))
except:
proto.write_pkt_line(None)
raise
if not 'delete-refs' in server_capabilities:
# Server does not support deletions. Fail later.
def remove_del(pair):
if pair[1] == ZERO_SHA:
if 'report-status' in negotiated_capabilities:
report_status_parser._ref_statuses.append(
'ng %s remote does not support deleting refs'
% pair[1])
report_status_parser._ref_status_ok = False
return False
else:
return True
new_refs = dict(
filter(
remove_del,
[(ref, sha) for ref, sha in new_refs.iteritems()]))
if new_refs is None:
proto.write_pkt_line(None)
return old_refs
if len(new_refs) == 0 and len(orig_new_refs):
# NOOP - Original new refs filtered out by policy
proto.write_pkt_line(None)
if self._report_status_parser is not None:
self._report_status_parser.check()
return old_refs
(have, want) = self._handle_receive_pack_head(proto,
negotiated_capabilities, old_refs, new_refs)
if not want and old_refs == new_refs:
return new_refs
objects = generate_pack_contents(have, want)
if len(objects) > 0:
entries, sha = write_pack_objects(proto.write_file(), objects)
elif len(set(new_refs.values()) - set([ZERO_SHA])) > 0:
# Check for valid create/update refs
filtered_new_refs = \
dict([(ref, sha) for ref, sha in new_refs.iteritems()
if sha != ZERO_SHA])
if len(set(filtered_new_refs.iteritems()) -
set(old_refs.iteritems())) > 0:
entries, sha = write_pack_objects(proto.write_file(), objects)
self._handle_receive_pack_tail(proto, negotiated_capabilities,
progress)
return new_refs
print(e.__dict__)
print(hexd(e.sha()))
map_commit(hexd(e.sha()))
out_list = []
for e in OBJ_LIST:
sha = hexd(e.sha())
if isinstance(e, objects.Blob):
e = BLOB_STORE[BLOB_MAP[sha]]
elif isinstance(e, objects.Tree):
e = TREE_STORE[TREE_MAP[sha]]
elif isinstance(e, objects.Commit):
e = COMMIT_STORE[COMMIT_MAP[sha]]
out_list.append((e, None))
f = open("crypt.pack", "wb")
obj_map, check_sha = pack.write_pack_objects(f, out_list)
f.close()
pack_sha = binascii.hexlify(check_sha).decode("ascii")
os.rename("crypt.pack", "crypt-%s.pack" % pack_sha)
entries = [(k, v[0], v[1]) for (k, v) in obj_map.items()]
entries.sort()
with open("crypt-%s.idx" % pack_sha, "wb") as f:
pack.write_pack_index_v2(f, entries, check_sha)
def send_pack(self,
path,
determine_wants,
generate_pack_contents,
progress=None):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of the
shas of the objects to upload.
:param progress: Optional callback called with progress updates
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
proto, unused_can_read = self._connect('receive-pack', path)
old_refs, server_capabilities = self._read_refs(proto)
negotiated_capabilities = self._send_capabilities & server_capabilities
if 'report-status' in negotiated_capabilities:
self._report_status_parser = ReportStatusParser()
report_status_parser = self._report_status_parser
try:
new_refs = orig_new_refs = determine_wants(dict(old_refs))
except:
proto.write_pkt_line(None)
raise
if not 'delete-refs' in server_capabilities:
# Server does not support deletions. Fail later.
def remove_del(pair):
if pair[1] == ZERO_SHA:
if 'report-status' in negotiated_capabilities:
report_status_parser._ref_statuses.append(
'ng %s remote does not support deleting refs' %
pair[1])
report_status_parser._ref_status_ok = False
return False
else:
return True
new_refs = dict(
filter(remove_del,
[(ref, sha) for ref, sha in new_refs.iteritems()]))
if new_refs is None:
proto.write_pkt_line(None)
return old_refs
if len(new_refs) == 0 and len(orig_new_refs):
# NOOP - Original new refs filtered out by policy
proto.write_pkt_line(None)
if self._report_status_parser is not None:
self._report_status_parser.check()
return old_refs
(have, want) = self._handle_receive_pack_head(proto,
negotiated_capabilities,
old_refs, new_refs)
if not want and old_refs == new_refs:
return new_refs
objects = generate_pack_contents(have, want)
if len(objects) > 0:
entries, sha = write_pack_objects(proto.write_file(), objects)
elif len(set(new_refs.values()) - set([ZERO_SHA])) > 0:
# Check for valid create/update refs
filtered_new_refs = \
dict([(ref, sha) for ref, sha in new_refs.iteritems()
if sha != ZERO_SHA])
if len(
set(filtered_new_refs.iteritems()) -
set(old_refs.iteritems())) > 0:
entries, sha = write_pack_objects(proto.write_file(), objects)
self._handle_receive_pack_tail(proto, negotiated_capabilities,
progress)
return new_refs
def test_add_pack(self):
o = DiskObjectStore(self.store_dir)
f, commit = o.add_pack()
b = make_object(Blob, data="more yummy data")
write_pack_objects(f, [(b, None)])
commit()
