def handle(self):
write = lambda x: self.proto.write_sideband(1, x)
graph_walker = ProtocolGraphWalker(self, self.repo.object_store,
self.repo.get_peeled)
objects_iter = self.repo.fetch_objects(graph_walker.determine_wants,
graph_walker,
self.progress,
get_tagged=self.get_tagged)
# Do they want any objects?
if len(objects_iter) == 0:
return
#self.progress("dul-daemon says what\n")
self.progress("counting objects: %d, done.\n" % len(objects_iter))
write_pack_data(ProtocolFile(None, write), objects_iter,
len(objects_iter))
#self.progress("how was that, then?\n")
size = self.backend.repository.info.size
msg = _('Repository size: %s\n' % filesizeformat(size))
logging.info(msg)
self.progress(msg)
# we are done
self.proto.write("0000")
def handle(self):
def determine_wants(heads):
keys = heads.keys()
if keys:
self.proto.write_pkt_line("%s %s\x00%s\n" % ( heads[keys[0]], keys[0], self.capabilities()))
for k in keys[1:]:
self.proto.write_pkt_line("%s %s\n" % (heads[k], k))
# i'm done..
self.proto.write("0000")
# Now client will either send "0000", meaning that it doesnt want to pull.
# or it will start sending want want want commands
want = self.proto.read_pkt_line()
if want == None:
return []
want, self.client_capabilities = extract_capabilities(want)
want_revs = []
while want and want[:4] == 'want':
want_revs.append(want[5:45])
want = self.proto.read_pkt_line()
return want_revs
progress = lambda x: self.proto.write_sideband(2, x)
write = lambda x: self.proto.write_sideband(1, x)
class ProtocolGraphWalker(object):
def __init__(self, proto):
self.proto = proto
self._last_sha = None
def ack(self, have_ref):
self.proto.write_pkt_line("ACK %s continue\n" % have_ref)
def next(self):
have = self.proto.read_pkt_line()
if have[:4] == 'have':
return have[5:45]
#if have[:4] == 'done':
# return None
if self._last_sha:
# Oddness: Git seems to resend the last ACK, without the "continue" statement
self.proto.write_pkt_line("ACK %s\n" % self._last_sha)
# The exchange finishes with a NAK
self.proto.write_pkt_line("NAK\n")
graph_walker = ProtocolGraphWalker(self.proto)
(num_objects, objects_iter) = self.backend.fetch_objects(determine_wants, graph_walker, progress)
progress("dul-daemon says what\n")
progress("counting objects: %d, done.\n" % len(objects))
write_pack_data(ProtocolFile(None, write), objects_iter, num_objects)
progress("how was that, then?\n")
# we are done
self.proto.write("0000")
def add_objects(self, objects):
"""Add a set of objects to this object store.
:param objects: Iterable over objects, should support __len__.
"""
if len(objects) == 0:
# Don't bother writing an empty pack file
return
f, commit = self.add_pack()
write_pack_data(f, objects, len(objects))
commit()
def add_objects(self, objects):
"""Add a set of objects to this object store.
:param objects: Iterable over objects, should support __len__.
"""
if len(objects) == 0:
# Don't bother writing an empty pack file
return
f, commit = self.add_pack()
write_pack_data(f, objects, len(objects))
commit()
def add_pack_data(self, count, pack_data, progress=None):
"""Add pack data to this object store.
:param num_items: Number of items to add
:param pack_data: Iterator over pack data tuples
"""
if count == 0:
# Don't bother writing an empty pack file
return
f, commit, abort = self.add_pack()
try:
write_pack_data(f, count, pack_data, progress)
except BaseException:
abort()
raise
else:
return commit()
def add_pack_data(self, count, pack_data, progress=None):
"""Add pack data to this object store.
:param num_items: Number of items to add
:param pack_data: Iterator over pack data tuples
"""
if count == 0:
# Don't bother writing an empty pack file
return
f, commit, abort = self.add_pack()
try:
write_pack_data(f, count, pack_data, progress)
except BaseException:
abort()
raise
else:
return commit()
def handle(self):
write = lambda x: self.proto.write_sideband(1, x)
graph_walker = ProtocolGraphWalker(self, self.repo.object_store, self.repo.get_peeled)
objects_iter = self.repo.fetch_objects(
graph_walker.determine_wants, graph_walker, self.progress, get_tagged=self.get_tagged
)
# Did the process short-circuit (e.g. in a stateless RPC call)? Note
# that the client still expects a 0-object pack in most cases.
if objects_iter is None:
return
self.progress("dul-daemon says what\n")
self.progress("counting objects: %d, done.\n" % len(objects_iter))
write_pack_data(ProtocolFile(None, write), objects_iter, len(objects_iter))
self.progress("how was that, then?\n")
# we are done
self.proto.write("0000")
def test_send_pack_new_ref(self):
self.rin.write(
b'0064310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/master\x00 report-status delete-refs ofs-delta\n'
b'0000000eunpack ok\n'
b'0019ok refs/heads/blah12\n'
b'0000')
self.rin.seek(0)
tree = Tree()
commit = Commit()
commit.tree = tree
commit.parents = []
commit.author = commit.committer = b'test user'
commit.commit_time = commit.author_time = 1174773719
commit.commit_timezone = commit.author_timezone = 0
commit.encoding = b'UTF-8'
commit.message = b'test message'
def update_refs(refs):
return {
b'refs/heads/blah12': commit.id,
b'refs/heads/master':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_data(have, want, ofs_delta=False):
return pack_objects_to_data([
(commit, None),
(tree, b''),
])
f = BytesIO()
write_pack_data(f, *generate_pack_data(None, None))
self.client.send_pack(b'/', update_refs, generate_pack_data)
self.assertIn(self.rout.getvalue(), [
b'007f0000000000000000000000000000000000000000 ' + commit.id +
b' refs/heads/blah12\x00report-status ofs-delta0000' +
f.getvalue(), b'007f0000000000000000000000000000000000000000 ' +
commit.id + b' refs/heads/blah12\x00ofs-delta report-status0000' +
f.getvalue()
])
def handle(self):
progress = lambda x: self.proto.write_sideband(2, x)
write = lambda x: self.proto.write_sideband(1, x)
graph_walker = ProtocolGraphWalker(self)
objects_iter = self.backend.fetch_objects(
graph_walker.determine_wants, graph_walker, progress)
# Do they want any objects?
if len(objects_iter) == 0:
return
progress("dul-daemon says what\n")
progress("counting objects: %d, done.\n" % len(objects_iter))
write_pack_data(ProtocolFile(None, write), objects_iter,
len(objects_iter))
progress("how was that, then?\n")
# we are done
self.proto.write("0000")
def test_send_pack_new_ref(self):
self.rin.write(
b'0064310ca9477129b8586fa2afc779c1f57cf64bba6c '
b'refs/heads/master\x00 report-status delete-refs ofs-delta\n'
b'0000000eunpack ok\n'
b'0019ok refs/heads/blah12\n'
b'0000')
self.rin.seek(0)
tree = Tree()
commit = Commit()
commit.tree = tree
commit.parents = []
commit.author = commit.committer = b'test user'
commit.commit_time = commit.author_time = 1174773719
commit.commit_timezone = commit.author_timezone = 0
commit.encoding = b'UTF-8'
commit.message = b'test message'
def determine_wants(refs):
return {
b'refs/heads/blah12': commit.id,
b'refs/heads/master':
b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
}
def generate_pack_data(have, want, ofs_delta=False):
return pack_objects_to_data([(commit, None), (tree, b''), ])
f = BytesIO()
write_pack_data(f, *generate_pack_data(None, None))
self.client.send_pack(b'/', determine_wants, generate_pack_data)
self.assertIn(
self.rout.getvalue(),
[b'007f0000000000000000000000000000000000000000 ' + commit.id +
b' refs/heads/blah12\x00report-status ofs-delta0000' +
f.getvalue(),
b'007f0000000000000000000000000000000000000000 ' + commit.id +
b' refs/heads/blah12\x00ofs-delta report-status0000' +
f.getvalue()])
def send_pack(self, path, determine_wants, generate_pack_contents):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return the shas of the
objects to upload.
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
proto, unused_can_read = self._connect('receive-pack', path)
old_refs, server_capabilities = self.read_refs(proto)
if 'report-status' not in server_capabilities:
self._send_capabilities.remove('report-status')
new_refs = determine_wants(old_refs)
if not new_refs:
proto.write_pkt_line(None)
return {}
want = []
have = [x for x in old_refs.values() if not x == ZERO_SHA]
sent_capabilities = False
for refname in set(new_refs.keys() + old_refs.keys()):
old_sha1 = old_refs.get(refname, ZERO_SHA)
new_sha1 = new_refs.get(refname, ZERO_SHA)
if old_sha1 != new_sha1:
if sent_capabilities:
proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
refname))
else:
proto.write_pkt_line(
'%s %s %s\0%s' % (old_sha1, new_sha1, refname,
' '.join(self._send_capabilities)))
sent_capabilities = True
if new_sha1 not in have and new_sha1 != ZERO_SHA:
want.append(new_sha1)
proto.write_pkt_line(None)
if not want:
return new_refs
objects = generate_pack_contents(have, want)
entries, sha = write_pack_data(proto.write_file(), objects,
len(objects))
if 'report-status' in self._send_capabilities:
self._parse_status_report(proto)
# wait for EOF before returning
data = proto.read()
if data:
raise SendPackError('Unexpected response %r' % data)
return new_refs
def handle(self):
write = lambda x: self.proto.write_sideband(1, x)
graph_walker = ProtocolGraphWalker(self, self.repo.object_store,
self.repo.get_peeled)
objects_iter = self.repo.fetch_objects(
graph_walker.determine_wants, graph_walker, self.progress,
get_tagged=self.get_tagged)
# Do they want any objects?
if len(objects_iter) == 0:
return
#self.progress("dul-daemon says what\n")
self.progress("counting objects: %d, done.\n" % len(objects_iter))
write_pack_data(ProtocolFile(None, write), objects_iter,
len(objects_iter))
#self.progress("how was that, then?\n")
size = self.backend.repository.info.size
msg = _('Repository size: %s\n' % filesizeformat(size))
logging.info(msg)
self.progress(msg)
# we are done
self.proto.write("0000")
def send_pack(self, path, determine_wants, generate_pack_contents):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return the shas of the
objects to upload.
"""
old_refs, server_capabilities = self.read_refs()
new_refs = determine_wants(old_refs)
if not new_refs:
self.proto.write_pkt_line(None)
return {}
want = []
have = [x for x in old_refs.values() if not x == ZERO_SHA]
sent_capabilities = False
for refname in set(new_refs.keys() + old_refs.keys()):
old_sha1 = old_refs.get(refname, ZERO_SHA)
new_sha1 = new_refs.get(refname, ZERO_SHA)
if old_sha1 != new_sha1:
if sent_capabilities:
self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, refname))
else:
self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, refname, self.capabilities()))
sent_capabilities = True
if not new_sha1 in (have, ZERO_SHA):
want.append(new_sha1)
self.proto.write_pkt_line(None)
if not want:
return new_refs
objects = generate_pack_contents(have, want)
(entries, sha) = write_pack_data(self.proto.write_file(), objects,
len(objects))
# read the final confirmation sha
try:
client_sha = self.proto.read_pkt_line()
except HangupException:
# for git-daemon versions before v1.6.6.1-26-g38a81b4, there is
# nothing to read; catch this and hide from the user.
pass
else:
if not client_sha in (None, "", sha):
raise ChecksumMismatch(sha, client_sha)
return new_refs
def send_pack(self, path, determine_wants, generate_pack_contents):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return the shas of the
objects to upload.
"""
old_refs, server_capabilities = self.read_refs()
new_refs = determine_wants(old_refs)
if not new_refs:
self.proto.write_pkt_line(None)
return {}
want = []
have = [x for x in old_refs.values() if not x == "0" * 40]
sent_capabilities = False
for refname in set(new_refs.keys() + old_refs.keys()):
old_sha1 = old_refs.get(refname, "0" * 40)
new_sha1 = new_refs.get(refname, "0" * 40)
if old_sha1 != new_sha1:
if sent_capabilities:
self.proto.write_pkt_line("%s %s %s" %
(old_sha1, new_sha1, refname))
else:
self.proto.write_pkt_line(
"%s %s %s\0%s" %
(old_sha1, new_sha1, refname, self.capabilities()))
sent_capabilities = True
if not new_sha1 in (have, "0" * 40):
want.append(new_sha1)
self.proto.write_pkt_line(None)
if not want:
return new_refs
objects = generate_pack_contents(have, want)
(entries, sha) = write_pack_data(self.proto.write_file(), objects,
len(objects))
# read the final confirmation sha
client_sha = self.proto.read(20)
if not client_sha in (None, "", sha):
raise ChecksumMismatch(sha, client_sha)
return new_refs
def send_pack(self, path, determine_wants, generate_pack_contents):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return the shas of the
objects to upload.
"""
old_refs, server_capabilities = self.read_refs()
new_refs = determine_wants(old_refs)
if not new_refs:
self.proto.write_pkt_line(None)
return {}
want = []
have = [x for x in old_refs.values() if not x == "0" * 40]
sent_capabilities = False
for refname in set(new_refs.keys() + old_refs.keys()):
old_sha1 = old_refs.get(refname, "0" * 40)
new_sha1 = new_refs.get(refname, "0" * 40)
if old_sha1 != new_sha1:
if sent_capabilities:
self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, refname))
else:
self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, refname, self.capabilities()))
sent_capabilities = True
if not new_sha1 in (have, "0" * 40):
want.append(new_sha1)
self.proto.write_pkt_line(None)
if not want:
return new_refs
objects = generate_pack_contents(have, want)
(entries, sha) = write_pack_data(self.proto.write_file(), objects,
len(objects))
# read the final confirmation sha
client_sha = self.proto.read(20)
if not client_sha in (None, "", sha):
raise ChecksumMismatch(sha, client_sha)
return new_refs
def test_add_thin_pack(self):
o = DiskObjectStore(self.store_dir)
f, commit = o.add_thin_pack()
b = make_object(Blob, data="more yummy data")
write_pack_data(f, [(b, None)], 1)
commit()