def test_send_pack_from_shallow_clone(self):
c = self._client()
server_new_path = os.path.join(self.gitroot, "server_new.export")
run_git_or_fail(["config", "http.uploadpack", "true"],
cwd=server_new_path)
run_git_or_fail(["config", "http.receivepack", "true"],
cwd=server_new_path)
remote_path = self._build_path("/server_new.export")
with repo.Repo(self.dest) as local:
result = c.fetch(remote_path, local, depth=1)
for r in result.refs.items():
local.refs.set_if_equals(r[0], None, r[1])
tree_id = local[local.head()].tree
for filename, contents in [
("bar", "bar contents"),
("zop", "zop contents"),
]:
tree_id = self._add_file(local, tree_id, filename, contents)
commit_id = local.do_commit(
message=b"add " + filename.encode("utf-8"),
committer=b"Joe Example <*****@*****.**>",
tree=tree_id,
)
sendrefs = dict(local.get_refs())
del sendrefs[b"HEAD"]
c.send_pack(remote_path, lambda _: sendrefs,
local.generate_pack_data)
with repo.Repo(server_new_path) as remote:
self.assertEqual(remote.head(), commit_id)
def _CheckoutLatestVersion(target_dir, url):
"""Pull tags and checkout the latest version of the target directory.
Args:
target_dir: (str) Directory name.
url: (str) Git repository URL.
Raises:
errors.HangupException: Hangup during communication to a remote repository.
"""
local_repo = repo.Repo(target_dir)
try:
# We don't get the tags with a clone or a fetch, so we have to get them
# after the fact.
client_wrapper = WrapClient(url)
local_repo = repo.Repo(target_dir)
tag, revision = _PullTags(local_repo, client_wrapper, target_dir)
log.info('Checking out revision [%s] of [%s] into [%s]', tag, url,
target_dir)
try:
# Checkout the specified revision of the runtime definition from git.
index.build_index_from_tree(local_repo.path, local_repo.index_path(),
local_repo.object_store,
revision.tree)
except (IOError, OSError, WindowsError) as ex:
raise InvalidTargetDirectoryError(
'Unable to checkout directory {0}: {1}'.format(target_dir,
ex.message))
finally:
local_repo.close()
def test_get_refs(self):
c = self._client()
refs = c.get_refs(self._build_path('/server_new.export'))
repo_dir = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(repo_dir) as dest:
self.assertDictEqual(dest.refs.as_dict(), refs)
def disable_ff_and_make_dummy_commit(self):
# disable non-fast-forward pushes to the server
dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
run_git_or_fail(['config', 'receive.denyNonFastForwards', 'true'],
cwd=dest.path)
commit_id = self.make_dummy_commit(dest)
return dest, commit_id
def test_fetch_pack_no_side_band_64k(self):
c = self._client()
c._fetch_capabilities.remove('side-band-64k')
dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
refs = c.fetch(self._build_path('/server_new.export'), dest)
map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
self.assertDestEqualsSrc()
def test_fetch_pack(self):
c = self._client()
with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
result = c.fetch(self._build_path('/server_new.export'), dest)
for r in result.refs.items():
dest.refs.set_if_equals(r[0], None, r[1])
self.assertDestEqualsSrc()
def gen_issue_numbers_from_git(
*,
commit_count: int,
base_sha: str,
head_sha: str,
) -> Iterator[Tuple[str, List[int]]]:
from dulwich import repo
r = repo.Repo(".")
walker = r.get_walker(max_entries=commit_count + 2, reverse=True)
for entry in walker:
sha = entry.commit.id.decode("utf8")
message = entry.commit.message.decode("utf8")
issue_numbers = [int(num) for num in ISSUE_NUMBER_RE.findall(message)]
if base_sha:
if base_sha == sha:
base_sha = ""
continue
print(
f"Checked {sha}: {len(issue_numbers)} issue numbers",
file=sys.stderr,
)
print(message, end="\n\n")
yield (sha, issue_numbers)
if sha == head_sha:
break
def _do_send_pack(self):
c = self._client()
srcpath = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(srcpath) as src:
sendrefs = dict(src.get_refs())
del sendrefs[b'HEAD']
c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
src.object_store.generate_pack_data)
def test_fetch_pack_zero_sha(self):
# zero sha1s are already present on the client, and should
# be ignored
c = self._client()
dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
refs = c.fetch(self._build_path('/server_new.export'), dest,
lambda refs: [protocol.ZERO_SHA])
map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
def test_fetch_pack_no_side_band_64k(self):
c = self._client()
c._fetch_capabilities.remove(b'side-band-64k')
with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
result = c.fetch(self._build_path('/server_new.export'), dest)
for r in result.refs.items():
dest.refs.set_if_equals(r[0], None, r[1])
self.assertDestEqualsSrc()
def test_incremental_fetch_pack(self):
self.test_fetch_pack()
dest, dummy = self.disable_ff_and_make_dummy_commit()
dest.refs['refs/heads/master'] = dummy
c = self._client()
dest = repo.Repo(os.path.join(self.gitroot, 'server_new.export'))
refs = c.fetch(self._build_path('/dest'), dest)
map(lambda r: dest.refs.set_if_equals(r[0], None, r[1]), refs.items())
self.assertDestEqualsSrc()
def test_send_without_report_status(self):
c = self._client()
c._send_capabilities.remove(b'report-status')
srcpath = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(srcpath) as src:
sendrefs = dict(src.get_refs())
del sendrefs[b'HEAD']
c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
src.object_store.generate_pack_data)
self.assertDestEqualsSrc()
def clean_path(self):
try:
repo.Repo(self.cleaned_data['path'])
except dulwich.errors.NotGitRepository:
raise forms.ValidationError(
_('Please submit a valid git repository path'))
except Exception:
raise forms.ValidationError(_('Please submit a valid file path'))
return self.cleaned_data['path']
def get_git_commit(filename):
"""Return author and email from last commit to filename."""
r = repo.Repo(".")
w = r.get_walker(paths=[filename], max_entries=1)
try:
c = next(iter(w)).commit
except StopIteration:
raise ValueError("No file %s anywhere in history." % filename)
return c
def _do_send_pack(self):
c = self._client()
srcpath = os.path.join(self.gitroot, "server_new.export")
with repo.Repo(srcpath) as src:
sendrefs = dict(src.get_refs())
del sendrefs[b"HEAD"]
c.send_pack(
self._build_path("/dest"),
lambda _: sendrefs,
src.generate_pack_data,
)
def test_fetch_pack_depth(self):
c = self._client()
with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
result = c.fetch(self._build_path('/server_new.export'), dest,
depth=1)
for r in result.refs.items():
dest.refs.set_if_equals(r[0], None, r[1])
self.assertEqual(
dest.get_shallow(),
set([b'35e0b59e187dd72a0af294aedffc213eaa4d03ff',
b'514dc6d3fbfe77361bcaef320c4d21b72bc10be9']))
def test_incremental_fetch_pack(self):
self.test_fetch_pack()
dest, dummy = self.disable_ff_and_make_dummy_commit()
dest.refs[b'refs/heads/master'] = dummy
c = self._client()
repo_dir = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(repo_dir) as dest:
result = c.fetch(self._build_path('/dest'), dest)
for r in result.refs.items():
dest.refs.set_if_equals(r[0], None, r[1])
self.assertDestEqualsSrc()
def test_send_pack_one_error(self):
dest, dummy_commit = self.disable_ff_and_make_dummy_commit()
dest.refs[b'refs/heads/master'] = dummy_commit
repo_dir = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(repo_dir) as src:
sendrefs, gen_pack = self.compute_send(src)
c = self._client()
result = c.send_pack(
self._build_path('/dest'), lambda _: sendrefs, gen_pack)
self.assertEqual({b'refs/heads/branch': None,
b'refs/heads/master': 'non-fast-forward'},
result.ref_status)
def test_fetch_pack_zero_sha(self):
# zero sha1s are already present on the client, and should
# be ignored
c = self._client()
with repo.Repo(os.path.join(self.gitroot, "dest")) as dest:
result = c.fetch(
self._build_path("/server_new.export"),
dest,
lambda refs, **kwargs: [protocol.ZERO_SHA],
)
for r in result.refs.items():
dest.refs.set_if_equals(r[0], None, r[1])
def test_send_without_report_status(self):
c = self._client()
c._send_capabilities.remove(b"report-status")
srcpath = os.path.join(self.gitroot, "server_new.export")
with repo.Repo(srcpath) as src:
sendrefs = dict(src.get_refs())
del sendrefs[b"HEAD"]
c.send_pack(
self._build_path("/dest"),
lambda _: sendrefs,
src.generate_pack_data,
)
self.assertDestEqualsSrc()
def test_send_remove_branch(self):
dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
dummy_commit = self.make_dummy_commit(dest)
dest.refs['refs/heads/master'] = dummy_commit
dest.refs['refs/heads/abranch'] = dummy_commit
sendrefs = dict(dest.refs)
sendrefs['refs/heads/abranch'] = "00" * 20
del sendrefs['HEAD']
gen_pack = lambda have, want: []
c = self._client()
self.assertEquals(dest.refs["refs/heads/abranch"], dummy_commit)
c.send_pack(self._build_path('/dest'), lambda _: sendrefs, gen_pack)
self.assertFalse("refs/heads/abranch" in dest.refs)
def test_send_pack_multiple_errors(self):
dest, dummy = self.disable_ff_and_make_dummy_commit()
# set up for two non-ff errors
branch, master = b'refs/heads/branch', b'refs/heads/master'
dest.refs[branch] = dest.refs[master] = dummy
repo_dir = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(repo_dir) as src:
sendrefs, gen_pack = self.compute_send(src)
c = self._client()
result = c.send_pack(
self._build_path('/dest'), lambda _: sendrefs, gen_pack)
self.assertEqual({branch: 'non-fast-forward',
master: 'non-fast-forward'},
result.ref_status)
def test_send_new_branch_empty_pack(self):
with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
dummy_commit = self.make_dummy_commit(dest)
dest.refs[b'refs/heads/master'] = dummy_commit
dest.refs[b'refs/heads/abranch'] = dummy_commit
sendrefs = {b'refs/heads/bbranch': dummy_commit}
def gen_pack(have, want, ofs_delta=False):
return 0, []
c = self._client()
self.assertEqual(dest.refs[b"refs/heads/abranch"], dummy_commit)
c.send_pack(
self._build_path('/dest'), lambda _: sendrefs, gen_pack)
self.assertEqual(dummy_commit, dest.refs[b"refs/heads/abranch"])
def test_push_remove_branch(self):
def determine_wants(*args):
return {
"refs/heads/pullr-108": objects.ZERO_SHA,
"refs/heads/master": local_repo.refs['refs/heads/master'],
"refs/heads/mybranch": local_repo.refs['refs/heads/mybranch'],
}
self.test_push_multiple_branch()
local_repo = repo.Repo(self.temp_d)
tcp_client = client.TCPGitClient(self.server_address, port=self.port)
tcp_client.send_pack(self.fakerepo, determine_wants,
local_repo.object_store.generate_pack_contents)
swift_repo = swift.SwiftRepo("fakerepo", self.conf)
self.assertNotIn('refs/heads/pullr-108', swift_repo.refs.allkeys())
def test_send_pack_one_error(self):
dest, dummy_commit = self.disable_ff_and_make_dummy_commit()
dest.refs[b'refs/heads/master'] = dummy_commit
repo_dir = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(repo_dir) as src:
sendrefs, gen_pack = self.compute_send(src)
c = self._client()
try:
c.send_pack(self._build_path(b'/dest'), lambda _: sendrefs, gen_pack)
except errors.UpdateRefsError as e:
self.assertEqual('refs/heads/master failed to update',
e.args[0])
self.assertEqual({b'refs/heads/branch': b'ok',
b'refs/heads/master': b'non-fast-forward'},
e.ref_status)
def test_send_remove_branch(self):
with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
dummy_commit = self.make_dummy_commit(dest)
dest.refs[b'refs/heads/master'] = dummy_commit
dest.refs[b'refs/heads/abranch'] = dummy_commit
sendrefs = dict(dest.refs)
sendrefs[b'refs/heads/abranch'] = b"00" * 20
del sendrefs[b'HEAD']
def gen_pack(have, want, ofs_delta=False):
return 0, []
c = self._client()
self.assertEqual(dest.refs[b"refs/heads/abranch"], dummy_commit)
c.send_pack(
self._build_path('/dest'), lambda _: sendrefs, gen_pack)
self.assertFalse(b"refs/heads/abranch" in dest.refs)
def test_fetch_empty_pack(self):
c = self._client()
with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
result = c.fetch(self._build_path('/server_new.export'), dest)
for r in result.refs.items():
dest.refs.set_if_equals(r[0], None, r[1])
self.assertDestEqualsSrc()
def dw(refs):
return list(refs.values())
result = c.fetch(
self._build_path('/server_new.export'), dest,
determine_wants=dw)
for r in result.refs.items():
dest.refs.set_if_equals(r[0], None, r[1])
self.assertDestEqualsSrc()
def test_send_pack_multiple_errors(self):
dest, dummy = self.disable_ff_and_make_dummy_commit()
# set up for two non-ff errors
branch, master = b'refs/heads/branch', b'refs/heads/master'
dest.refs[branch] = dest.refs[master] = dummy
repo_dir = os.path.join(self.gitroot, 'server_new.export')
with repo.Repo(repo_dir) as src:
sendrefs, gen_pack = self.compute_send(src)
c = self._client()
try:
c.send_pack(self._build_path(b'/dest'), lambda _: sendrefs, gen_pack)
except errors.UpdateRefsError as e:
self.assertIn(str(e),
['{0}, {1} failed to update'.format(
branch.decode('ascii'), master.decode('ascii')),
'{1}, {0} failed to update'.format(
branch.decode('ascii'), master.decode('ascii'))])
self.assertEqual({branch: b'non-fast-forward',
master: b'non-fast-forward'},
e.ref_status)
def commit_circles_to_git(repo_path='/tmp/circles/', committer=b''):
if os.path.exists(repo_path):
rep = repo.Repo(repo_path)
else:
rep = repo.Repo.init(repo_path, True)
with rep:
try:
head = rep.head()
except KeyError:
last_time = 0
else:
last_commit = rep[rep.head()]
last_time = last_commit.commit_time
last_time = make_aware(datetime.fromtimestamp(last_time))
qset = Circle.objects.filter(modified__gt=last_time)
index = rep.open_index()
num_changed = 0
ser = get_serializer('yaml')()
commit_message = []
for obj in qset:
data = ser.serialize([obj]).encode('utf8')
blob = objects.Blob.from_string(data)
rep.object_store.add_object(blob)
entry = (obj.created.timestamp(), obj.modified.timestamp(), 1, 1, 33188, 100, 100, len(data), blob.id, 0)
index[obj.get_path()] = entry
num_changed += 1
commit_message.append('Changed {}'.format(obj))
if num_changed:
tree = index.commit(rep.object_store)
message = '\n'.join(commit_message).encode('utf8')
print(rep.do_commit(
message=message,
author=committer,
committer=committer,
commit_timestamp=now().timestamp(),
tree=tree))
def assertDestEqualsSrc(self):
repo_dir = os.path.join(self.gitroot, 'server_new.export')
dest_repo_dir = os.path.join(self.gitroot, 'dest')
with repo.Repo(repo_dir) as src:
with repo.Repo(dest_repo_dir) as dest:
self.assertReposEqual(src, dest)
