def get(self, repo):
service = self.request.get('service')
if service:
handler_cls = DEFAULT_HANDLERS.get(service)
if not handler_cls:
self.error(403)
return
self.response.headers['Content-type'] = 'application/x-%s-advertisement' % service
proto = ReceivableProtocol(StringIO().read, self.response.out.write)
handler = handler_cls(FileSystemBackend(), [ repo ], proto, stateless_rpc = True, advertise_refs = True)
handler.proto.write_pkt_line('# service=%s\n' % service)
handler.proto.write_pkt_line(None)
handler.handle()
else:
self.response.headers['Content-type'] = 'text/plain'
repo = Repo(repo)
refs = repo.get_refs()
for name in sorted(refs.iterkeys()):
if name == 'HEAD':
continue
sha = refs[name]
if not repo[sha]:
continue
self.response.out.write('%s\t%s\n' % (sha, name))
peeled_sha = repo.get_peeled(name)
if peeled_sha != sha:
self.response.out.write('%s\t%s^{}\n' % (peeled_sha, name))
def get_recent_tags(projdir=PROJDIR):
"""
Get list of recent tags in order from newest to oldest and their datetimes.
:param projdir: path to ``.git``
:returns: list of (tag, [datetime, commit, author]) sorted from new to old
"""
project = Repo(projdir) # dulwich repository object
refs = project.get_refs() # dictionary of refs and their SHA-1 values
tags = {} # empty dictionary to hold tags, commits and datetimes
# iterate over refs in repository
for key, value in refs.iteritems():
obj = project.get_object(value) # dulwich object from SHA-1
# check if object is tag
if obj.type_name != 'tag':
# skip ref if not a tag
continue
# strip the leading text from "refs/tag/<tag name>" to get "tag name"
_, tag = key.rsplit('/', 1)
# check if tag object is commit, altho it should always be true
if obj.object[0].type_name == 'commit':
commit = project.get_object(obj.object[1]) # commit object
# get tag commit datetime, but dulwich returns seconds since
# beginning of epoch, so use Python time module to convert it to
# timetuple then convert to datetime
tags[tag] = [
datetime.datetime(*time.gmtime(commit.commit_time)[:6]),
commit.id,
commit.author
]
# return list of tags sorted by their datetimes from newest to oldest
return sorted(tags.iteritems(), key=lambda tag: tag[1][0], reverse=True)
def list_all_contributors (self):
tmp = 1
tot = len(self.repos)
all_contribs = []
for repo in self.repos:
print >> sys.stderr, "[%d/%d Analyzing %s]" % (tmp, tot, repo)
tmp += 1
repo = Repo(repo)
master = repo.get_refs()['refs/heads/master']
for i in repo.get_walker ([master]):
if "<" in i.commit.author:
split = i.commit.author.split("<")
author = split[0]
email = split[1]
author = author.strip ()
email = email.strip ()
email = email[:-1]
else:
author = i.commit.author
email = ""
all_contribs.append((author, email))
del repo
tmp = []
for c in all_contribs:
if c in tmp:
continue
tmp.append(c)
return tmp
def get_refs(repo: Repo) -> DefaultDict[str, Set[str]]:
refs: DefaultDict[str, Set[str]] = defaultdict(set)
for ref, commit in repo.get_refs().items():
str_commit = _to_str(commit)
refs[str_commit].add(str_commit)
refs[str_commit].add(_to_str(ref))
return refs
def get_tags(repo: Repo) -> Dict[str, str]:
tags: Dict[str, str] = {}
for tag_ref, commit in repo.get_refs().items():
tag_ref_str = _to_str(tag_ref)
if tag_ref_str.startswith("refs/tags/") and VALID_TAG.match(
tag_ref_str[len("refs/tags/"):]):
tags[_to_str(commit)] = os.path.basename(tag_ref_str)
return tags
def test_simple(self):
self.build_tree(['remote/foo'])
self.remote_tree.add("foo")
revid = self.remote_tree.commit("msg")
git_sha1 = map_to_git_sha1(self.remote_dir, revid)
out = self.fetch([(git_sha1, 'HEAD')])
self.assertEqual(out, b"\n")
r = Repo('local')
self.assertTrue(git_sha1 in r.object_store)
self.assertEqual({}, r.get_refs())
def repo_dashboard(repo_key, branch, tree_path=''):
#Get repo & branch
repo = Repo(settings.REPOS[repo_key])
try:
branch_or_sha = repo.get_refs()[LOCAL_BRANCH_PREFIX+branch]
except KeyError:
#If there is no branch then is a commit sha
branch_or_sha = branch
branch_name = branch
selected_commit = repo[branch_or_sha]
#Get files
tree_files={}
splitted_path = tree_path.split('/')
#Get filename (maybe we need fo the blob)
file_name = splitted_path[len(splitted_path)-1]
#if we are root folder then empty list
if len(splitted_path) == 1 and splitted_path[0] =='':
splitted_path = []
else:
splitted_path.reverse()
tree = repo[selected_commit.tree]
#for tree_file in tree.iteritems():
# tree_files[tree_file] = stat.S_ISDIR(tree_file[1])
tree_files = utils.get_repo_files(repo, tree, splitted_path)
#Show file content if is a blob (raw file, not dir)
if isinstance(tree_files, Blob):
file_code = tree_files.as_raw_string()
return render_template('file-detail.html', repo_key=repo_key, branch=branch_name,
file_code=file_code, file_name=file_name)
#Show tree if is a empty path (root folder) or a subforlder
else:
#Check readme
readme = ''
readme_name = ''
for maybe_readme, directory in tree_files.iteritems():
if 'readme' in maybe_readme[0].lower():
readme_name = maybe_readme[0]
readme = repo[maybe_readme[2]].as_raw_string()
break
#Little hack for the url generating
tree_path = tree_path + '/' if tree_path is not '' else tree_path
return render_template('repo-dashboard.html', repo_key=repo_key, branch=branch_name,
tree_files=tree_files, readme=readme, readme_name=readme_name,
tree_path=tree_path)
def _find_date_boundaries (self):
for repo in self.repos:
repo = Repo(repo)
master = repo.get_refs()['refs/heads/master']
for i in repo.get_walker ([master]):
if self.date_oldest == None or self.date_oldest > i.commit.commit_time:
self.date_oldest = i.commit.commit_time
if self.date_newest == None or self.date_newest < i.commit.commit_time:
self.date_newest = i.commit.commit_time
del i
del repo
def commit_history(repo_key, branch):
repo = Repo(settings.REPOS[repo_key])
#get all the branches and set the name branch in a ref list (don't
#add the selected one, this will be added sepparetly in the template)
references = []
selected_branch = branch
for ref, sha in repo.get_refs().iteritems():
#get the name of the branch without the pefix
if (LOCAL_BRANCH_PREFIX in ref):
references.append(ref.replace(LOCAL_BRANCH_PREFIX, '', 1))
#Get the branch walker
walker = repo.get_walker(include = [repo.get_refs()[LOCAL_BRANCH_PREFIX+branch], ])
#Start getting all the commits from the branch
commits = []
commits_per_day = []
previous_commit_time = None
#Group commits by day (I use list instead of a dict because the list is ordered already, so I don't need to sort the dict)
for i in walker:
commit = i.commit
commit_time = filters.convert_unix_time_filter(commit.commit_time, '%d %b %Y')
#if is new or like the previous one time, then add to the list, if not then save the list and create a new one
if (previous_commit_time is None) or (commit_time == previous_commit_time):
commits_per_day.append(commit)
else:
commits.append(commits_per_day)
commits_per_day = [commit,]
previous_commit_time = commit_time
#Add last ones
commits.append(commits_per_day)
return render_template('commit-history.html', commits=commits, repo_key=repo_key,
references = references, selected_branch=selected_branch)
def add_doc(namespace_path, project_path):
user_repo = os.path.join(namespace_path, project_path)
repo_path = build_repo_path(user_repo)
if not os.path.exists(repo_path):
err_msg = "Repo path {0} not exist".format(repo_path)
print >> sys.stderr, err_msg
return
ix = open_dir(INDEXDIR)
writer = ix.writer()
try:
repo = Repo(repo_path)
except NotGitRepository:
err_msg = "No git repository was found at {0}".format(repo_path)
print >> sys.stderr, err_msg
return
try:
refs = repo.get_refs()
for ref in refs.keys():
if ref.startswith('refs/heads') or ref.startswith('refs/tags'):
obj = repo.get_object(refs[ref])
if isinstance(obj, Tag):
commit = repo[obj.object[1]]
else:
commit = obj
tree = repo[commit.tree]
for path, entry in get_entry(repo, tree):
filename = os.path.join(ref.rsplit('/')[2],
path, entry.path)
blob = repo[entry.sha]
writer.add_document(
repo=user_repo.decode('UTF-8', 'ignore'),
ref=ref.decode('UTF-8', 'ignore'),
filename=filename.decode('UTF-8', 'ignore'),
content=blob.data.decode('UTF-8', 'ignore'))
writer.commit()
except:
writer.cancel()
raise
class GitVCS(object):
def __init__(self, path):
self.repo = Repo(path)
def branches(self):
return dict(((k.replace('refs/heads/', ''), v) for k, v in self.repo.get_refs().items()
if k.startswith('refs/heads/')))
def get_branch_ref(self, branch):
branches = self.branches()
if not branch in branches:
return None
return branches[branch]
def get_object(self, sha1):
try:
return self.repo.get_object(sha1)
except AssertionError:
return None
def get_object_by_path(self, ref, path):
c = self.get_object(self.get_branch_ref(ref) or ref)
if not c:
return None
paths = path.split(os.sep)
count = len(paths)
obj = self.get_object(c.tree)
for i, x in enumerate(paths):
if not x:
break
try:
_mode, sha1 = obj[x]
except KeyError:
obj = None
break
obj = self.get_object(sha1)
if i < count - 1 and not isinstance(obj, Tree):
obj = None
break
if not obj:
raise ValueError("Bad path")
print "Result: ", type(obj), obj
return obj
def branches(self, **keywords):
sha, branch, filename = keywords["sha"], keywords["branch"], keywords["filename"]
repo = Repo(self.package.path())
data = repo.get_refs()
valid_keys = []
for key in data.keys():
if key[0:11] == "refs/heads/":
valid_keys.append((key[11:], data[key]))
content = ""
for key in valid_keys:
content = content + "<a href=\"/package/" + self.package.name + ":" + key[0] + "/\">" + key[0] + "</a><br />"
self.content = content
self.branch = branch
self.sha = sha
return self.respond()
def send_pack(self,
path,
determine_wants,
generate_pack_contents,
progress=None,
write_pack=write_pack_objects):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of
the shas of the objects to upload.
:param progress: Optional progress function
:param write_pack: Function called with (file, iterable of objects) to
write the objects returned by generate_pack_contents to the server.
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
from dulwich.repo import Repo
target = Repo(path)
old_refs = target.get_refs()
new_refs = determine_wants(old_refs)
have = [sha1 for sha1 in old_refs.values() if sha1 != ZERO_SHA]
want = []
for refname in set(new_refs.keys() + old_refs.keys()):
old_sha1 = old_refs.get(refname, ZERO_SHA)
new_sha1 = new_refs.get(refname, ZERO_SHA)
if new_sha1 not in have and new_sha1 != ZERO_SHA:
want.append(new_sha1)
if not want and old_refs == new_refs:
return new_refs
target.object_store.add_objects(generate_pack_contents(have, want))
for name, sha in new_refs.iteritems():
target.refs[name] = sha
return new_refs
class GitRepository(Repository):
CommitBuilder = GitCommitBuilder
def __init__(self, path=None, workdir=None, create=False, bare=False):
assert path or workdir
if workdir:
self.path = workdir.path
else:
self.path = path
if create:
# XXX: fragile
path.ensure(dir=1)
self.repo = Repo.init(path.strpath)
else:
assert self.path.check(dir=True)
try:
self.repo = Repo(self.path.strpath)
except NotGitRepository:
raise NotFoundError('git', self.path)
def __len__(self):
# XXX: fragile
head = self.get_default_head()
if head is None:
return 0
return len(self.repo.revision_history(head.id))
def push(self):
# XXX: hell, figure if the remote is empty, push master in that case
# XXX: use dulwich?
subprocess.check_call(['git', 'push', '--all'], cwd=self.path)
def get_default_head(self):
revs = self.repo.get_refs()
head = revs.get('HEAD', revs.get('master'))
if head is not None:
return GitRevision(self, self.repo.get_object(head))
def __getitem__(self, id):
return GitRevision(self, self.repo.get_object(id))
def send_pack(self, path, determine_wants, generate_pack_contents,
progress=None, write_pack=write_pack_objects):
"""Upload a pack to a remote repository.
:param path: Repository path
:param generate_pack_contents: Function that can return a sequence of
the shas of the objects to upload.
:param progress: Optional progress function
:param write_pack: Function called with (file, iterable of objects) to
write the objects returned by generate_pack_contents to the server.
:raises SendPackError: if server rejects the pack data
:raises UpdateRefsError: if the server supports report-status
and rejects ref updates
"""
from dulwich.repo import Repo
target = Repo(path)
old_refs = target.get_refs()
new_refs = determine_wants(old_refs)
have = [sha1 for sha1 in old_refs.values() if sha1 != ZERO_SHA]
want = []
all_refs = set(new_refs.keys()).union(set(old_refs.keys()))
for refname in all_refs:
old_sha1 = old_refs.get(refname, ZERO_SHA)
new_sha1 = new_refs.get(refname, ZERO_SHA)
if new_sha1 not in have and new_sha1 != ZERO_SHA:
want.append(new_sha1)
if not want and old_refs == new_refs:
return new_refs
target.object_store.add_objects(generate_pack_contents(have, want))
for name, sha in new_refs.items():
target.refs[name] = sha
return new_refs
def build_stats_by_periods (self, periods, filter_fn=None):
assert (len(periods) > 0)
assert (reduce(lambda x,y: x and y, map(lambda x: isinstance(x,int), periods)))
lower = self.date_oldest
upper = self.date_newest
periods.sort()
periods = dict.fromkeys(periods, [])
tmp = 1
tot = len(self.repos)
for repo in self.repos:
print >> sys.stderr, "[%d/%d Analyzing %s]" % (tmp, tot, repo)
tmp += 1
repo = Repo(repo)
master = repo.get_refs()['refs/heads/master']
for i in repo.get_walker ([master]):
keys = periods.keys()
keys.sort()
lower = keys[0]
upper = keys[-1]
if i.commit.commit_time < lower or i.commit.commit_time > upper:
continue
if filter_fn != None and not filter_fn (i):
continue
period = self._find_period (periods.keys(), i.commit.commit_time)
author = i.commit.author.split("<")[0].strip()
periods[period].append(author)
del i
del repo
for period in periods.keys():
periods[period] = self._plain_to_count(periods[period])
return periods
def push(remote_url, repo_path='.'):
"""
Push to a remote repository
:param remote_url: <str> url of remote repository
:param repo_path: <str> path of local repository
:return refs: <dict> dictionary of ref-sha pairs
"""
client, path = get_transport_and_path(remote_url)
r = Repo(repo_path)
objsto = r.object_store
refs = r.get_refs()
def update_refs(old):
# TODO: Too complicated, not necessary to find the refs that
# differ - it's fine to update a ref even if it already exists.
# TODO: Also error out if there are non-fast forward updates
same = list(set(refs).intersection(old))
new = dict([(k,refs[k]) for k in same if refs[k] != old[k]])
dfky = list(set(refs) - set(new))
dfrnt = dict([(k,refs[k]) for k in dfky if k != 'HEAD'])
return dict(new.items() + dfrnt.items())
return client.send_pack(path,
update_refs,
objsto.generate_pack_contents,
sys.stdout.write)
def find_lost_commits():
r=Repo('.')
o=r.object_store
all_commits=[]
all_trees=[]
for sha in o:
obj=o[sha]
if isinstance(obj,Commit):
all_commits.append(obj)
#hide commits that have children, so we get "ends" of chains only
for c in all_commits:
for p in c.parents:
try:
all_commits.remove(p)
except ValueError:
pass
if c.sha in r.get_refs().values():
all_commits.remove(c)
#hide commits that are in branches
for c in sorted(all_commits,key=lambda x:x.commit_time):
print('{} {} {}'.format(time.ctime(c.commit_time),sha, c.message))
def find_lost_commits():
r = Repo('.')
o = r.object_store
all_commits = []
all_trees = []
for sha in o:
obj = o[sha]
if isinstance(obj, Commit):
all_commits.append(obj)
#hide commits that have children, so we get "ends" of chains only
for c in all_commits:
for p in c.parents:
try:
all_commits.remove(p)
except ValueError:
pass
if c.sha in r.get_refs().values():
all_commits.remove(c)
#hide commits that are in branches
for c in sorted(all_commits, key=lambda x: x.commit_time):
print('{} {} {}'.format(time.ctime(c.commit_time), sha, c.message))
class DiskRefsContainerTests(RefsContainerTests, TestCase):
def setUp(self):
TestCase.setUp(self)
self._repo = open_repo('refs.git')
self.addCleanup(tear_down_repo, self._repo)
self._refs = self._repo.refs
def test_get_packed_refs(self):
self.assertEqual(
{
b'refs/heads/packed':
b'42d06bd4b77fed026b154d16493e5deab78f02ec',
b'refs/tags/refs-0.1':
b'df6800012397fb85c56e7418dd4eb9405dee075c',
}, self._refs.get_packed_refs())
def test_get_peeled_not_packed(self):
# not packed
self.assertEqual(None, self._refs.get_peeled(b'refs/tags/refs-0.2'))
self.assertEqual(b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
self._refs[b'refs/tags/refs-0.2'])
# packed, known not peelable
self.assertEqual(self._refs[b'refs/heads/packed'],
self._refs.get_peeled(b'refs/heads/packed'))
# packed, peeled
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.get_peeled(b'refs/tags/refs-0.1'))
def test_setitem(self):
RefsContainerTests.test_setitem(self)
path = os.path.join(self._refs.path, b'refs', b'some', b'ref')
with open(path, 'rb') as f:
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
f.read()[:40])
self.assertRaises(OSError, self._refs.__setitem__,
b'refs/some/ref/sub',
b'42d06bd4b77fed026b154d16493e5deab78f02ec')
def test_setitem_packed(self):
with open(os.path.join(self._refs.path, b'packed-refs'), 'w') as f:
f.write('# pack-refs with: peeled fully-peeled sorted \n')
f.write(
'42d06bd4b77fed026b154d16493e5deab78f02ec refs/heads/packed\n')
# It's allowed to set a new ref on a packed ref, the new ref will be
# placed outside on refs/
self._refs[b'refs/heads/packed'] = (
b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8')
packed_ref_path = os.path.join(self._refs.path, b'refs', b'heads',
b'packed')
with open(packed_ref_path, 'rb') as f:
self.assertEqual(b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
f.read()[:40])
self.assertRaises(OSError, self._refs.__setitem__,
b'refs/heads/packed/sub',
b'42d06bd4b77fed026b154d16493e5deab78f02ec')
def test_setitem_symbolic(self):
ones = b'1' * 40
self._refs[b'HEAD'] = ones
self.assertEqual(ones, self._refs[b'HEAD'])
# ensure HEAD was not modified
f = open(os.path.join(self._refs.path, b'HEAD'), 'rb')
v = next(iter(f)).rstrip(b'\n\r')
f.close()
self.assertEqual(b'ref: refs/heads/master', v)
# ensure the symbolic link was written through
f = open(os.path.join(self._refs.path, b'refs', b'heads', b'master'),
'rb')
self.assertEqual(ones, f.read()[:40])
f.close()
def test_set_if_equals(self):
RefsContainerTests.test_set_if_equals(self)
# ensure symref was followed
self.assertEqual(b'9' * 40, self._refs[b'refs/heads/master'])
# ensure lockfile was deleted
self.assertFalse(
os.path.exists(
os.path.join(self._refs.path, b'refs', b'heads',
b'master.lock')))
self.assertFalse(
os.path.exists(os.path.join(self._refs.path, b'HEAD.lock')))
def test_add_if_new_packed(self):
# don't overwrite packed ref
self.assertFalse(
self._refs.add_if_new(b'refs/tags/refs-0.1', b'9' * 40))
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
def test_add_if_new_symbolic(self):
# Use an empty repo instead of the default.
repo_dir = os.path.join(tempfile.mkdtemp(), 'test')
os.makedirs(repo_dir)
repo = Repo.init(repo_dir)
self.addCleanup(tear_down_repo, repo)
refs = repo.refs
nines = b'9' * 40
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertFalse(b'refs/heads/master' in refs)
self.assertTrue(refs.add_if_new(b'HEAD', nines))
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
self.assertFalse(refs.add_if_new(b'HEAD', b'1' * 40))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
def test_follow(self):
self.assertEqual(([b'HEAD', b'refs/heads/master'
], b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs.follow(b'HEAD'))
self.assertEqual(([b'refs/heads/master'
], b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs.follow(b'refs/heads/master'))
self.assertRaises(KeyError, self._refs.follow, b'refs/heads/loop')
def test_delitem(self):
RefsContainerTests.test_delitem(self)
ref_file = os.path.join(self._refs.path, b'refs', b'heads', b'master')
self.assertFalse(os.path.exists(ref_file))
self.assertFalse(b'refs/heads/master' in self._refs.get_packed_refs())
def test_delitem_symbolic(self):
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
del self._refs[b'HEAD']
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs[b'refs/heads/master'])
self.assertFalse(os.path.exists(os.path.join(self._refs.path,
b'HEAD')))
def test_remove_if_equals_symref(self):
# HEAD is a symref, so shouldn't equal its dereferenced value
self.assertFalse(
self._refs.remove_if_equals(
b'HEAD', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertTrue(
self._refs.remove_if_equals(
b'refs/heads/master',
b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/heads/master'])
# HEAD is now a broken symref
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
self.assertFalse(
os.path.exists(
os.path.join(self._refs.path, b'refs', b'heads',
b'master.lock')))
self.assertFalse(
os.path.exists(os.path.join(self._refs.path, b'HEAD.lock')))
def test_remove_packed_without_peeled(self):
refs_file = os.path.join(self._repo.path, 'packed-refs')
f = GitFile(refs_file)
refs_data = f.read()
f.close()
f = GitFile(refs_file, 'wb')
f.write(b'\n'.join(l for l in refs_data.split(b'\n')
if not l or l[0] not in b'#^'))
f.close()
self._repo = Repo(self._repo.path)
refs = self._repo.refs
self.assertTrue(
refs.remove_if_equals(b'refs/heads/packed',
b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
def test_remove_if_equals_packed(self):
# test removing ref that is only packed
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
self.assertTrue(
self._refs.remove_if_equals(
b'refs/tags/refs-0.1',
b'df6800012397fb85c56e7418dd4eb9405dee075c'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/tags/refs-0.1'])
def test_remove_parent(self):
self._refs[b'refs/heads/foo/bar'] = (
b'df6800012397fb85c56e7418dd4eb9405dee075c')
del self._refs[b'refs/heads/foo/bar']
ref_file = os.path.join(
self._refs.path,
b'refs',
b'heads',
b'foo',
b'bar',
)
self.assertFalse(os.path.exists(ref_file))
ref_file = os.path.join(self._refs.path, b'refs', b'heads', b'foo')
self.assertFalse(os.path.exists(ref_file))
ref_file = os.path.join(self._refs.path, b'refs', b'heads')
self.assertTrue(os.path.exists(ref_file))
self._refs[b'refs/heads/foo'] = (
b'df6800012397fb85c56e7418dd4eb9405dee075c')
def test_read_ref(self):
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_ref(b'HEAD'))
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.read_ref(b'refs/heads/packed'))
self.assertEqual(None, self._refs.read_ref(b'nonexistant'))
def test_read_loose_ref(self):
self._refs[b'refs/heads/foo'] = (
b'df6800012397fb85c56e7418dd4eb9405dee075c')
self.assertEqual(None, self._refs.read_ref(b'refs/heads/foo/bar'))
def test_non_ascii(self):
try:
encoded_ref = u'refs/tags/schön'.encode(
sys.getfilesystemencoding())
except UnicodeEncodeError:
raise SkipTest(
"filesystem encoding doesn't support special character")
p = os.path.join(self._repo.path.encode(sys.getfilesystemencoding()),
encoded_ref)
with open(p, 'w') as f:
f.write('00' * 20)
expected_refs = dict(_TEST_REFS)
expected_refs[encoded_ref] = b'00' * 20
del expected_refs[b'refs/heads/loop']
self.assertEqual(expected_refs, self._repo.get_refs())
def test_cyrillic(self):
if sys.platform == 'win32':
raise SkipTest(
"filesystem encoding doesn't support arbitrary bytes")
# reported in https://github.com/dulwich/dulwich/issues/608
name = b'\xcd\xee\xe2\xe0\xff\xe2\xe5\xf2\xea\xe01'
encoded_ref = b'refs/heads/' + name
with open(
os.path.join(
self._repo.path.encode(sys.getfilesystemencoding()),
encoded_ref), 'w') as f:
f.write('00' * 20)
expected_refs = set(_TEST_REFS.keys())
expected_refs.add(encoded_ref)
self.assertEqual(expected_refs, set(self._repo.refs.allkeys()))
self.assertEqual(
{
r[len(b'refs/'):]
for r in expected_refs if r.startswith(b'refs/')
}, set(self._repo.refs.subkeys(b'refs/')))
expected_refs.remove(b'refs/heads/loop')
expected_refs.add(b'HEAD')
self.assertEqual(expected_refs, set(self._repo.get_refs().keys()))
class DiskRefsContainerTests(RefsContainerTests, TestCase):
def setUp(self):
TestCase.setUp(self)
self._repo = open_repo('refs.git')
self.addCleanup(tear_down_repo, self._repo)
self._refs = self._repo.refs
def test_get_packed_refs(self):
self.assertEqual({
b'refs/heads/packed': b'42d06bd4b77fed026b154d16493e5deab78f02ec',
b'refs/tags/refs-0.1': b'df6800012397fb85c56e7418dd4eb9405dee075c',
}, self._refs.get_packed_refs())
def test_get_peeled_not_packed(self):
# not packed
self.assertEqual(None, self._refs.get_peeled(b'refs/tags/refs-0.2'))
self.assertEqual(b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
self._refs[b'refs/tags/refs-0.2'])
# packed, known not peelable
self.assertEqual(self._refs[b'refs/heads/packed'],
self._refs.get_peeled(b'refs/heads/packed'))
# packed, peeled
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.get_peeled(b'refs/tags/refs-0.1'))
def test_setitem(self):
RefsContainerTests.test_setitem(self)
path = os.path.join(self._refs.path, b'refs', b'some', b'ref')
with open(path, 'rb') as f:
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
f.read()[:40])
self.assertRaises(
OSError, self._refs.__setitem__,
b'refs/some/ref/sub', b'42d06bd4b77fed026b154d16493e5deab78f02ec')
def test_setitem_packed(self):
with open(os.path.join(self._refs.path, b'packed-refs'), 'w') as f:
f.write('# pack-refs with: peeled fully-peeled sorted \n')
f.write(
'42d06bd4b77fed026b154d16493e5deab78f02ec refs/heads/packed\n')
# It's allowed to set a new ref on a packed ref, the new ref will be
# placed outside on refs/
self._refs[b'refs/heads/packed'] = (
b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8'
)
packed_ref_path = os.path.join(
self._refs.path, b'refs', b'heads', b'packed')
with open(packed_ref_path, 'rb') as f:
self.assertEqual(
b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
f.read()[:40])
self.assertRaises(
OSError, self._refs.__setitem__,
b'refs/heads/packed/sub',
b'42d06bd4b77fed026b154d16493e5deab78f02ec')
def test_setitem_symbolic(self):
ones = b'1' * 40
self._refs[b'HEAD'] = ones
self.assertEqual(ones, self._refs[b'HEAD'])
# ensure HEAD was not modified
f = open(os.path.join(self._refs.path, b'HEAD'), 'rb')
v = next(iter(f)).rstrip(b'\n\r')
f.close()
self.assertEqual(b'ref: refs/heads/master', v)
# ensure the symbolic link was written through
f = open(os.path.join(self._refs.path, b'refs', b'heads', b'master'),
'rb')
self.assertEqual(ones, f.read()[:40])
f.close()
def test_set_if_equals(self):
RefsContainerTests.test_set_if_equals(self)
# ensure symref was followed
self.assertEqual(b'9' * 40, self._refs[b'refs/heads/master'])
# ensure lockfile was deleted
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, b'refs', b'heads', b'master.lock')))
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, b'HEAD.lock')))
def test_add_if_new_packed(self):
# don't overwrite packed ref
self.assertFalse(self._refs.add_if_new(b'refs/tags/refs-0.1',
b'9' * 40))
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
def test_add_if_new_symbolic(self):
# Use an empty repo instead of the default.
repo_dir = os.path.join(tempfile.mkdtemp(), 'test')
os.makedirs(repo_dir)
repo = Repo.init(repo_dir)
self.addCleanup(tear_down_repo, repo)
refs = repo.refs
nines = b'9' * 40
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertFalse(b'refs/heads/master' in refs)
self.assertTrue(refs.add_if_new(b'HEAD', nines))
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
self.assertFalse(refs.add_if_new(b'HEAD', b'1' * 40))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
def test_follow(self):
self.assertEqual(([b'HEAD', b'refs/heads/master'],
b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs.follow(b'HEAD'))
self.assertEqual(([b'refs/heads/master'],
b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs.follow(b'refs/heads/master'))
self.assertRaises(KeyError, self._refs.follow, b'refs/heads/loop')
def test_delitem(self):
RefsContainerTests.test_delitem(self)
ref_file = os.path.join(self._refs.path, b'refs', b'heads', b'master')
self.assertFalse(os.path.exists(ref_file))
self.assertFalse(b'refs/heads/master' in self._refs.get_packed_refs())
def test_delitem_symbolic(self):
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
del self._refs[b'HEAD']
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs[b'refs/heads/master'])
self.assertFalse(
os.path.exists(os.path.join(self._refs.path, b'HEAD')))
def test_remove_if_equals_symref(self):
# HEAD is a symref, so shouldn't equal its dereferenced value
self.assertFalse(self._refs.remove_if_equals(
b'HEAD', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertTrue(self._refs.remove_if_equals(
b'refs/heads/master', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/heads/master'])
# HEAD is now a broken symref
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, b'refs', b'heads', b'master.lock')))
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, b'HEAD.lock')))
def test_remove_packed_without_peeled(self):
refs_file = os.path.join(self._repo.path, 'packed-refs')
f = GitFile(refs_file)
refs_data = f.read()
f.close()
f = GitFile(refs_file, 'wb')
f.write(b'\n'.join(l for l in refs_data.split(b'\n')
if not l or l[0] not in b'#^'))
f.close()
self._repo = Repo(self._repo.path)
refs = self._repo.refs
self.assertTrue(refs.remove_if_equals(
b'refs/heads/packed', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
def test_remove_if_equals_packed(self):
# test removing ref that is only packed
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
self.assertTrue(
self._refs.remove_if_equals(
b'refs/tags/refs-0.1',
b'df6800012397fb85c56e7418dd4eb9405dee075c'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/tags/refs-0.1'])
def test_remove_parent(self):
self._refs[b'refs/heads/foo/bar'] = (
b'df6800012397fb85c56e7418dd4eb9405dee075c'
)
del self._refs[b'refs/heads/foo/bar']
ref_file = os.path.join(
self._refs.path, b'refs', b'heads', b'foo', b'bar',
)
self.assertFalse(os.path.exists(ref_file))
ref_file = os.path.join(self._refs.path, b'refs', b'heads', b'foo')
self.assertFalse(os.path.exists(ref_file))
ref_file = os.path.join(self._refs.path, b'refs', b'heads')
self.assertTrue(os.path.exists(ref_file))
self._refs[b'refs/heads/foo'] = (
b'df6800012397fb85c56e7418dd4eb9405dee075c'
)
def test_read_ref(self):
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_ref(b'HEAD'))
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.read_ref(b'refs/heads/packed'))
self.assertEqual(None, self._refs.read_ref(b'nonexistant'))
def test_read_loose_ref(self):
self._refs[b'refs/heads/foo'] = (
b'df6800012397fb85c56e7418dd4eb9405dee075c'
)
self.assertEqual(None, self._refs.read_ref(b'refs/heads/foo/bar'))
def test_non_ascii(self):
try:
encoded_ref = u'refs/tags/schön'.encode(
sys.getfilesystemencoding())
except UnicodeEncodeError:
raise SkipTest(
"filesystem encoding doesn't support special character")
p = os.path.join(
self._repo.path.encode(sys.getfilesystemencoding()),
encoded_ref)
with open(p, 'w') as f:
f.write('00' * 20)
expected_refs = dict(_TEST_REFS)
expected_refs[encoded_ref] = b'00' * 20
del expected_refs[b'refs/heads/loop']
self.assertEqual(expected_refs, self._repo.get_refs())
def test_cyrillic(self):
if sys.platform == 'win32':
raise SkipTest(
"filesystem encoding doesn't support arbitrary bytes")
# reported in https://github.com/dulwich/dulwich/issues/608
name = b'\xcd\xee\xe2\xe0\xff\xe2\xe5\xf2\xea\xe01'
encoded_ref = b'refs/heads/' + name
with open(os.path.join(
self._repo.path.encode(
sys.getfilesystemencoding()), encoded_ref), 'w') as f:
f.write('00' * 20)
expected_refs = set(_TEST_REFS.keys())
expected_refs.add(encoded_ref)
self.assertEqual(expected_refs,
set(self._repo.refs.allkeys()))
self.assertEqual({r[len(b'refs/'):] for r in expected_refs
if r.startswith(b'refs/')},
set(self._repo.refs.subkeys(b'refs/')))
expected_refs.remove(b'refs/heads/loop')
expected_refs.add(b'HEAD')
self.assertEqual(expected_refs,
set(self._repo.get_refs().keys()))
class GitHandler(object):
mapfile = 'git-mapfile'
tagsfile = 'git-tags'
def __init__(self, dest_repo, ui):
self.repo = dest_repo
self.ui = ui
if ui.configbool('git', 'intree'):
self.gitdir = self.repo.wjoin('.git')
else:
self.gitdir = self.repo.join('git')
self.init_author_file()
self.paths = ui.configitems('paths')
self.branch_bookmark_suffix = ui.config('git', 'branch_bookmark_suffix')
self._map_git_real = {}
self._map_hg_real = {}
self.load_tags()
@property
def _map_git(self):
if not self._map_git_real:
self.load_map()
return self._map_git_real
@property
def _map_hg(self):
if not self._map_hg_real:
self.load_map()
return self._map_hg_real
# make the git data directory
def init_if_missing(self):
if os.path.exists(self.gitdir):
self.git = Repo(self.gitdir)
else:
os.mkdir(self.gitdir)
self.git = Repo.init_bare(self.gitdir)
def init_author_file(self):
self.author_map = {}
if self.ui.config('git', 'authors'):
f = open(self.repo.wjoin(
self.ui.config('git', 'authors')))
try:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
from_, to = RE_AUTHOR_FILE.split(line, 2)
self.author_map[from_] = to
finally:
f.close()
## FILE LOAD AND SAVE METHODS
def map_set(self, gitsha, hgsha):
self._map_git[gitsha] = hgsha
self._map_hg[hgsha] = gitsha
def map_hg_get(self, gitsha):
return self._map_git.get(gitsha)
def map_git_get(self, hgsha):
return self._map_hg.get(hgsha)
def load_map(self):
if os.path.exists(self.repo.join(self.mapfile)):
for line in self.repo.opener(self.mapfile):
gitsha, hgsha = line.strip().split(' ', 1)
self._map_git_real[gitsha] = hgsha
self._map_hg_real[hgsha] = gitsha
def save_map(self):
file = self.repo.opener(self.mapfile, 'w+', atomictemp=True)
for hgsha, gitsha in sorted(self._map_hg.iteritems()):
file.write("%s %s\n" % (gitsha, hgsha))
# If this complains that NoneType is not callable, then
# atomictempfile no longer has either of rename (pre-1.9) or
# close (post-1.9)
getattr(file, 'rename', getattr(file, 'close', None))()
def load_tags(self):
self.tags = {}
if os.path.exists(self.repo.join(self.tagsfile)):
for line in self.repo.opener(self.tagsfile):
sha, name = line.strip().split(' ', 1)
self.tags[name] = sha
def save_tags(self):
file = self.repo.opener(self.tagsfile, 'w+', atomictemp=True)
for name, sha in sorted(self.tags.iteritems()):
if not self.repo.tagtype(name) == 'global':
file.write("%s %s\n" % (sha, name))
# If this complains that NoneType is not callable, then
# atomictempfile no longer has either of rename (pre-1.9) or
# close (post-1.9)
getattr(file, 'rename', getattr(file, 'close', None))()
## END FILE LOAD AND SAVE METHODS
## COMMANDS METHODS
def import_commits(self, remote_name):
self.import_git_objects(remote_name)
self.update_hg_bookmarks(self.git.get_refs())
self.save_map()
def fetch(self, remote, heads):
self.export_commits()
refs = self.fetch_pack(remote, heads)
remote_name = self.remote_name(remote)
oldrefs = self.git.get_refs()
if refs:
self.import_git_objects(remote_name, refs)
self.import_tags(refs)
self.update_hg_bookmarks(refs)
if remote_name:
self.update_remote_branches(remote_name, refs)
elif not self.paths:
# intial cloning
self.update_remote_branches('default', refs)
# "Activate" a tipmost bookmark.
bms = getattr(self.repo['tip'], 'bookmarks',
lambda : None)()
if bms:
bookmarks.setcurrent(self.repo, bms[0])
def remoteref(ref):
rn = remote_name or 'default'
return 'refs/remotes/' + rn + ref[10:]
modheads = [refs[k] for k in refs if k.startswith('refs/heads/')
and not k.endswith('^{}')
and refs[k] != oldrefs.get(remoteref(k))]
if not modheads:
self.ui.status(_("no changes found\n"))
self.save_map()
return len(modheads)
def export_commits(self):
try:
self.export_git_objects()
self.export_hg_tags()
self.update_references()
finally:
self.save_map()
def get_refs(self, remote):
self.export_commits()
client, path = self.get_transport_and_path(remote)
old_refs = {}
new_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = set(self.local_heads().values() + self.tags.values())
new_refs.update(self.get_changed_refs(refs, to_push, True))
return refs # always return the same refs to make the send a no-op
try:
client.send_pack(path, changed, lambda have, want: [])
changed_refs = [ref for ref, sha in new_refs.iteritems()
if sha != old_refs.get(ref)]
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
old = {}
for r in old_refs:
old_ref = self.map_hg_get(old_refs[r])
if old_ref:
old[bin(old_ref)] = 1
return old, new
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
class Gittle(object):
"""All paths used in Gittle external methods must be paths relative to the git repository
"""
DEFAULT_COMMIT = "HEAD"
DEFAULT_BRANCH = "master"
DEFAULT_REMOTE = "origin"
DEFAULT_MESSAGE = "**No Message**"
DEFAULT_USER_INFO = {"name": None, "email": None}
DIFF_FUNCTIONS = {
"classic": utils.git.classic_tree_diff,
"dict": utils.git.dict_tree_diff,
"changes": utils.git.dict_tree_diff,
}
DEFAULT_DIFF_TYPE = "dict"
HIDDEN_REGEXES = [
# Hide git directory
r".*\/\.git\/.*"
]
# References
REFS_BRANCHES = "refs/heads/"
REFS_REMOTES = "refs/remotes/"
REFS_TAGS = "refs/tags/"
# Name pattern truths
# Used for detecting if files are :
# - deleted
# - added
# - changed
PATTERN_ADDED = (False, True)
PATTERN_REMOVED = (True, False)
PATTERN_MODIFIED = (True, True)
# Permissions
MODE_DIRECTORY = 040000 # Used to tell if a tree entry is a directory
# Tree depth
MAX_TREE_DEPTH = 1000
# Acceptable Root paths
ROOT_PATHS = (os.path.curdir, os.path.sep)
def __init__(self, repo_or_path, origin_uri=None, auth=None, report_activity=None, *args, **kwargs):
if isinstance(repo_or_path, DulwichRepo):
self.repo = repo_or_path
elif isinstance(repo_or_path, Gittle):
self.repo = DulwichRepo(repo_or_path.path)
elif isinstance(repo_or_path, basestring):
path = os.path.abspath(repo_or_path)
self.repo = DulwichRepo(path)
else:
logging.warning("Repo is of type %s" % type(repo_or_path))
raise Exception("Gittle must be initialized with either a dulwich repository or a string to the path")
# Set path
self.path = self.repo.path
# The remote url
self.origin_uri = origin_uri
# Report client activty
self._report_activity = report_activity
# Build ignore filter
self.hidden_regexes = copy.copy(self.HIDDEN_REGEXES)
self.hidden_regexes.extend(self._get_ignore_regexes())
self.ignore_filter = utils.paths.path_filter_regex(self.hidden_regexes)
self.filters = [self.ignore_filter]
# Get authenticator
if auth:
self.authenticator = auth
else:
self.auth(*args, **kwargs)
def report_activity(self, *args, **kwargs):
if not self._report_activity:
return
return self._report_activity(*args, **kwargs)
def _format_author(self, name, email):
return "%s <%s>" % (name, email)
def _format_userinfo(self, userinfo):
name = userinfo.get("name")
email = userinfo.get("email")
if name and email:
return self._format_author(name, email)
return None
def _format_ref(self, base, extra):
return "".join([base, extra])
def _format_ref_branch(self, branch_name):
return self._format_ref(self.REFS_BRANCHES, branch_name)
def _format_ref_remote(self, remote_name):
return self._format_ref(self.REFS_REMOTES, remote_name)
def _format_ref_tag(self, tag_name):
return self._format_ref(self.REFS_TAGS, tag_name)
@property
def head(self):
"""Return SHA of the current HEAD
"""
return self.repo.head()
@property
def is_bare(self):
"""Bare repositories have no working directories or indexes
"""
return self.repo.bare
@property
def is_working(self):
return not (self.is_bare)
def has_index(self):
"""Opposite of is_bare
"""
return self.repo.has_index()
@property
def has_commits(self):
"""
If the repository has no HEAD we consider that is has no commits
"""
try:
self.repo.head()
except KeyError:
return False
return True
def ref_walker(self, ref=None):
"""
Very simple, basic walker
"""
ref = ref or "HEAD"
sha = self._commit_sha(ref)
return self.repo.revision_history(sha)
def branch_walker(self, branch):
branch = branch or self.DEFAULT_BRANCH
ref = self._format_ref_branch(branch)
return self.ref_walker(ref)
def commit_info(self, start=0, end=None, branch=None):
"""Return a generator of commits with all their attached information
"""
if not self.has_commits:
return []
commits = [utils.git.commit_info(entry) for entry in self.branch_walker(branch)]
if not end:
return commits
return commits[start:end]
@funky.uniquify
def recent_contributors(self, n=None, branch=None):
n = n or 10
return funky.pluck(self.commit_info(end=n, branch=branch), "author")
@property
def commit_count(self):
try:
return len(self.ref_walker())
except KeyError:
return 0
def commits(self):
"""Return a list of SHAs for all the concerned commits
"""
return [commit["sha"] for commit in self.commit_info()]
@property
def git_dir(self):
return self.repo.controldir()
def auth(self, *args, **kwargs):
self.authenticator = GittleAuth(*args, **kwargs)
return self.authenticator
# Generate a branch selector (used for pushing)
def _wants_branch(self, branch_name=None):
branch_name = branch_name or self.DEFAULT_BRANCH
refs_key = self._format_ref_branch(branch_name)
sha = self.branches[branch_name]
def wants_func(old):
refs_key = self._format_ref_branch(branch_name)
return {refs_key: sha}
return wants_func
def _get_ignore_regexes(self):
gitignore_filename = os.path.join(self.path, ".gitignore")
if not os.path.exists(gitignore_filename):
return []
lines = open(gitignore_filename).readlines()
globers = map(lambda line: line.rstrip(), lines)
return utils.paths.globers_to_regex(globers)
# Get the absolute path for a file in the git repo
def abspath(self, repo_file):
return os.path.abspath(os.path.join(self.path, repo_file))
# Get the relative path from the absolute path
def relpath(self, abspath):
return os.path.relpath(abspath, self.path)
@property
def last_commit(self):
return self[self.repo.head()]
@property
def index(self):
return self.repo.open_index()
@classmethod
def init(cls, path, bare=None, *args, **kwargs):
"""Initialize a repository"""
mkdir_safe(path)
# Constructor to use
if bare:
constructor = DulwichRepo.init_bare
else:
constructor = DulwichRepo.init
# Create dulwich repo
repo = constructor(path)
# Create Gittle repo
return cls(repo, *args, **kwargs)
@classmethod
def init_bare(cls, *args, **kwargs):
kwargs.setdefault("bare", True)
return cls.init(*args, **kwargs)
def get_client(self, origin_uri=None, **kwargs):
# Get the remote URL
origin_uri = origin_uri or self.origin_uri
# Fail if inexistant
if not origin_uri:
raise InvalidRemoteUrl()
client_kwargs = {}
auth_kwargs = self.authenticator.kwargs()
client_kwargs.update(auth_kwargs)
client_kwargs.update(kwargs)
client_kwargs.update({"report_activity": self.report_activity})
client, remote_path = get_transport_and_path(origin_uri, **client_kwargs)
return client, remote_path
def push_to(self, origin_uri, branch_name=None, progress=None, progress_stderr=None):
selector = self._wants_branch(branch_name=branch_name)
client, remote_path = self.get_client(origin_uri, progress_stderr=progress_stderr)
return client.send_pack(remote_path, selector, self.repo.object_store.generate_pack_contents, progress=progress)
# Like: git push
def push(self, origin_uri=None, branch_name=None, progress=None, progress_stderr=None):
return self.push_to(origin_uri, branch_name, progress, progress_stderr)
# Not recommended at ALL ... !!!
def dirty_pull_from(self, origin_uri, branch_name=None):
# Remove all previously existing data
rmtree(self.path)
mkdir_safe(self.path)
self.repo = DulwichRepo.init(self.path)
# Fetch brand new copy from remote
return self.pull_from(origin_uri, branch_name)
def pull_from(self, origin_uri, branch_name=None):
return self.fetch(origin_uri)
# Like: git pull
def pull(self, origin_uri=None, branch_name=None):
return self.pull_from(origin_uri, branch_name)
def fetch_remote(self, origin_uri=None):
# Get client
client, remote_path = self.get_client(origin_uri=origin_uri)
# Fetch data from remote repository
remote_refs = client.fetch(remote_path, self.repo)
return remote_refs
def _setup_fetched_refs(self, refs, origin, bare):
remote_tags = utils.git.subrefs(refs, "refs/tags")
remote_heads = utils.git.subrefs(refs, "refs/heads")
# Filter refs
clean_remote_tags = utils.git.clean_refs(remote_tags)
clean_remote_heads = utils.git.clean_refs(remote_heads)
# Base of new refs
heads_base = "refs/remotes/" + origin
if bare:
heads_base = "refs/heads"
# Import branches
self.import_refs(heads_base, clean_remote_heads)
# Import tags
self.import_refs("refs/tags", clean_remote_tags)
# Update HEAD
self["HEAD"] = refs["HEAD"]
def fetch(self, origin_uri=None, bare=None, origin=None):
bare = bare or False
origin = origin or self.DEFAULT_REMOTE
# Remote refs
remote_refs = self.fetch_remote(origin_uri)
# Update head
# Hit repo because head doesn't yet exist so
# print("REFS = %s" % remote_refs)
# Update refs (branches, tags, HEAD)
self._setup_fetched_refs(remote_refs, origin, bare)
# Checkout working directories
if not bare:
self.checkout_all()
else:
self.update_server_info()
@classmethod
def clone(cls, origin_uri, local_path, auth=None, mkdir=True, bare=False, *args, **kwargs):
"""Clone a remote repository"""
mkdir_safe(local_path)
# Initialize the local repository
if bare:
local_repo = cls.init_bare(local_path)
else:
local_repo = cls.init(local_path)
repo = cls(local_repo, origin_uri=origin_uri, auth=auth, *args, **kwargs)
repo.fetch(bare=bare)
# Add origin
# TODO
return repo
@classmethod
def clone_bare(cls, *args, **kwargs):
"""Same as .clone except clones to a bare repository by default
"""
kwargs.setdefault("bare", True)
return cls.clone(*args, **kwargs)
def _commit(self, committer=None, author=None, message=None, files=None, tree=None, *args, **kwargs):
if not tree:
# If no tree then stage files
modified_files = files or self.modified_files
logging.warning("STAGING : %s" % modified_files)
self.add(modified_files)
# Messages
message = message or self.DEFAULT_MESSAGE
author_msg = self._format_userinfo(author)
committer_msg = self._format_userinfo(committer)
return self.repo.do_commit(
message=message, author=author_msg, committer=committer_msg, encoding="UTF-8", tree=tree, *args, **kwargs
)
def _tree_from_structure(self, structure):
# TODO : Support directories
tree = Tree()
for file_info in structure:
# str only
try:
data = file_info["data"].encode("ascii")
name = file_info["name"].encode("ascii")
mode = file_info["mode"]
except:
# Skip file on encoding errors
continue
blob = Blob()
blob.data = data
# Store file's contents
self.repo.object_store.add_object(blob)
# Add blob entry
tree.add(name, mode, blob.id)
# Store tree
self.repo.object_store.add_object(tree)
return tree.id
# Like: git commmit -a
def commit(self, name=None, email=None, message=None, files=None, *args, **kwargs):
user_info = {"name": name, "email": email}
return self._commit(committer=user_info, author=user_info, message=message, files=files, *args, **kwargs)
def commit_structure(self, name=None, email=None, message=None, structure=None, *args, **kwargs):
"""Main use is to do commits directly to bare repositories
For example doing a first Initial Commit so the repo can be cloned and worked on right away
"""
if not structure:
return
tree = self._tree_from_structure(structure)
user_info = {"name": name, "email": email}
return self._commit(committer=user_info, author=user_info, message=message, tree=tree, *args, **kwargs)
# Push all local commits
# and pull all remote commits
def sync(self, origin_uri=None):
self.push(origin_uri)
return self.pull(origin_uri)
def lookup_entry(self, relpath, trackable_files=set()):
if not relpath in trackable_files:
raise KeyError
abspath = self.abspath(relpath)
with open(abspath, "rb") as git_file:
data = git_file.read()
s = sha1()
s.update("blob %u\0" % len(data))
s.update(data)
return (s.hexdigest(), os.stat(abspath).st_mode)
@property
@funky.transform(set)
def tracked_files(self):
return list(self.index)
@property
@funky.transform(set)
def raw_files(self):
return utils.paths.subpaths(self.path)
@property
@funky.transform(set)
def ignored_files(self):
return utils.paths.subpaths(self.path, filters=self.filters)
@property
@funky.transform(set)
def trackable_files(self):
return self.raw_files - self.ignored_files
@property
@funky.transform(set)
def untracked_files(self):
return self.trackable_files - self.tracked_files
"""
@property
@funky.transform(set)
def modified_staged_files(self):
"Checks if the file has changed since last commit"
timestamp = self.last_commit.commit_time
index = self.index
return [
f
for f in self.tracked_files
if index[f][1][0] > timestamp
]
"""
# Return a list of tuples
# representing the changed elements in the git tree
def _changed_entries(self, ref=None):
ref = ref or self.DEFAULT_COMMIT
if not self.has_commits:
return []
obj_sto = self.repo.object_store
tree_id = self[ref].tree
names = self.trackable_files
lookup_func = partial(self.lookup_entry, trackable_files=names)
# Format = [((old_name, new_name), (old_mode, new_mode), (old_sha, new_sha)), ...]
tree_diff = changes_from_tree(names, lookup_func, obj_sto, tree_id, want_unchanged=False)
return list(tree_diff)
@funky.transform(set)
def _changed_entries_by_pattern(self, pattern):
changed_entries = self._changed_entries()
filtered_paths = [
funky.first_true(names)
for names, modes, sha in changed_entries
if tuple(map(bool, names)) == pattern and funky.first_true(names)
]
return filtered_paths
@property
@funky.transform(set)
def removed_files(self):
return self._changed_entries_by_pattern(self.PATTERN_REMOVED) - self.ignored_files
@property
@funky.transform(set)
def added_files(self):
return self._changed_entries_by_pattern(self.PATTERN_ADDED) - self.ignored_files
@property
@funky.transform(set)
def modified_files(self):
modified_files = self._changed_entries_by_pattern(self.PATTERN_MODIFIED) - self.ignored_files
return modified_files
@property
@funky.transform(set)
def modified_unstaged_files(self):
timestamp = self.last_commit.commit_time
return [f for f in self.tracked_files if os.stat(self.abspath(f)).st_mtime > timestamp]
@property
def pending_files(self):
"""
Returns a list of all files that could be possibly staged
"""
# Union of both
return self.modified_files | self.added_files | self.removed_files
@property
def pending_files_by_state(self):
files = {"modified": self.modified_files, "added": self.added_files, "removed": self.removed_files}
# "Flip" the dictionary
return {path: state for state, paths in files.items() for path in paths}
"""
@property
@funky.transform(set)
def modified_files(self):
return self.modified_staged_files | self.modified_unstaged_files
"""
# Like: git add
@funky.arglist_method
def stage(self, files):
return self.repo.stage(files)
def add(self, *args, **kwargs):
return self.stage(*args, **kwargs)
# Like: git rm
@funky.arglist_method
def rm(self, files, force=False):
index = self.index
index_files = filter(lambda f: f in index, files)
for f in index_files:
del self.index[f]
return index.write()
def mv_fs(self, file_pair):
old_name, new_name = file_pair
os.rename(old_name, new_name)
# Like: git mv
@funky.arglist_method
def mv(self, files_pair):
index = self.index
files_in_index = filter(lambda f: f[0] in index, files_pair)
map(self.mv_fs, files_in_index)
old_files = map(funky.first, files_in_index)
new_files = map(funky.last, files_in_index)
self.add(new_files)
self.rm(old_files)
self.add(old_files)
return
@working_only
def _checkout_tree(self, tree):
return build_index_from_tree(self.repo.path, self.repo.index_path(), self.repo.object_store, tree)
def checkout_all(self, commit_sha=None):
commit_sha = commit_sha or self.head
commit_tree = self._commit_tree(commit_sha)
# Rebuild index from the current tree
return self._checkout_tree(commit_tree)
def checkout(self, commit_sha=None, files=None):
"""Checkout only a select amount of files
"""
commit_sha = commit_sha or self.head
files = files or []
return self
@funky.arglist_method
def reset(self, files, commit="HEAD"):
pass
def rm_all(self):
self.index.clear()
return self.index.write()
def _to_commit(self, commit_obj):
"""Allows methods to accept both SHA's or dulwich Commit objects as arguments
"""
if isinstance(commit_obj, basestring):
return self.repo[commit_obj]
return commit_obj
def _commit_sha(self, commit_obj):
"""Extracts a Dulwich commits SHA
"""
if utils.git.is_sha(commit_obj):
return commit_obj
elif isinstance(commit_obj, basestring):
# Can't use self[commit_obj] to avoid infinite recursion
commit_obj = self.repo[commit_obj]
return commit_obj.id
def _blob_data(self, sha):
"""Return a blobs content for a given SHA
"""
return self[sha].data
# Get the nth parent back for a given commit
def get_parent_commit(self, commit, n=None):
""" Recursively gets the nth parent for a given commit
Warning: Remember that parents aren't the previous commits
"""
if n is None:
n = 1
commit = self._to_commit(commit)
parents = commit.parents
if n <= 0 or not parents:
# Return a SHA
return self._commit_sha(commit)
parent_sha = parents[0]
parent = self[parent_sha]
# Recur
return self.get_parent_commit(parent, n - 1)
def get_previous_commit(self, commit_ref, n=None):
commit_sha = self._parse_reference(commit_ref)
n = n or 1
commits = self.commits()
return funky.next(commits, commit_sha, n=n, default=commit_sha)
def _parse_reference(self, ref_string):
# COMMIT_REF~x
if "~" in ref_string:
ref, count = ref_string.split("~")
count = int(count)
commit_sha = self._commit_sha(ref)
return self.get_previous_commit(commit_sha, count)
return self._commit_sha(ref_string)
def _commit_tree(self, commit_sha):
"""Return the tree object for a given commit
"""
return self[commit_sha].tree
def diff(self, commit_sha, compare_to=None, diff_type=None, filter_binary=True):
diff_type = diff_type or self.DEFAULT_DIFF_TYPE
diff_func = self.DIFF_FUNCTIONS[diff_type]
if not compare_to:
compare_to = self.get_previous_commit(commit_sha)
return self._diff_between(compare_to, commit_sha, diff_function=diff_func)
def diff_working(self, ref=None, filter_binary=True):
"""Diff between the current working directory and the HEAD
"""
return utils.git.diff_changes_paths(
self.repo.object_store, self.path, self._changed_entries(ref=ref), filter_binary=filter_binary
)
def get_commit_files(self, commit_sha, parent_path=None, is_tree=None, paths=None):
"""Returns a dict of the following Format :
{
"directory/filename.txt": {
'name': 'filename.txt',
'path': "directory/filename.txt",
"sha": "xxxxxxxxxxxxxxxxxxxx",
"data": "blablabla",
"mode": 0xxxxx",
},
...
}
"""
# Default values
context = {}
is_tree = is_tree or False
parent_path = parent_path or ""
if is_tree:
tree = self[commit_sha]
else:
tree = self[self._commit_tree(commit_sha)]
for mode, path, sha in tree.entries():
# Check if entry is a directory
if mode == self.MODE_DIRECTORY:
context.update(
self.get_commit_files(sha, parent_path=os.path.join(parent_path, path), is_tree=True, paths=paths)
)
continue
subpath = os.path.join(parent_path, path)
# Only add the files we want
if not (paths is None or subpath in paths):
continue
# Add file entry
context[subpath] = {"name": path, "path": subpath, "mode": mode, "sha": sha, "data": self._blob_data(sha)}
return context
def file_versions(self, path):
"""Returns all commits where given file was modified
"""
versions = []
commits_info = self.commit_info()
seen_shas = set()
for commit in commits_info:
try:
files = self.get_commit_files(commit["sha"], paths=[path])
file_path, file_data = files.items()[0]
except IndexError:
continue
file_sha = file_data["sha"]
if file_sha in seen_shas:
continue
else:
seen_shas.add(file_sha)
# Add file info
commit["file"] = file_data
versions.append(file_data)
return versions
def _diff_between(self, old_commit_sha, new_commit_sha, diff_function=None, filter_binary=True):
"""Internal method for getting a diff between two commits
Please use .diff method unless you have very speciic needs
"""
# If commit is first commit (new_commit_sha == old_commit_sha)
# then compare to an empty tree
if new_commit_sha == old_commit_sha:
old_tree = Tree()
else:
old_tree = self._commit_tree(old_commit_sha)
new_tree = self._commit_tree(new_commit_sha)
return diff_function(self.repo.object_store, old_tree, new_tree, filter_binary=filter_binary)
def changes(self, *args, **kwargs):
""" List of changes between two SHAs
Returns a list of lists of tuples :
[
[
(oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
],
...
]
"""
kwargs["diff_type"] = "changes"
return self.diff(*args, **kwargs)
def changes_count(self, *args, **kwargs):
return len(self.changes(*args, **kwargs))
def _refs_by_pattern(self, pattern):
refs = self.refs
def item_filter(key_value):
"""Filter only concered refs"""
key, value = key_value
return key.startswith(pattern)
def item_map(key_value):
"""Rewrite keys"""
key, value = key_value
new_key = key[len(pattern) :]
return (new_key, value)
return dict(map(item_map, filter(item_filter, refs.items())))
@property
def refs(self):
return self.repo.get_refs()
def set_refs(refs_dict):
for k, v in refs_dict.items():
self.repo[k] = v
def import_refs(self, base, other):
return self.repo.refs.import_refs(base, other)
@property
def branches(self):
return self._refs_by_pattern(self.REFS_BRANCHES)
def _active_branch(self, refs=None, head=None):
head = head or self.head
refs = refs or self.branches
try:
return {branch: branch_head for branch, branch_head in refs.items() if branch_head == head}.items()[0]
except IndexError:
pass
return (None, None)
@property
def active_branch(self):
return self._active_branch()[0]
@property
def active_sha(self):
return self._active_branch()[1]
@property
def remote_branches(self):
return self._refs_by_pattern(self.REFS_REMOTES)
@property
def tags(self):
return self._refs_by_pattern(self.REFS_TAGS)
@property
def remotes(self):
""" Dict of remotes
{
'origin': 'http://friendco.de/some_user/repo.git',
...
}
"""
config = self.repo.get_config()
return {keys[1]: values["url"] for keys, values in config.items() if keys[0] == "remote"}
def add_ref(self, new_ref, old_ref):
self.repo.refs[new_ref] = self.repo.refs[old_ref]
self.update_server_info()
def remove_ref(self, ref_name):
# Returns False if ref doesn't exist
if not ref_name in self.repo.refs:
return False
del self.repo.refs[ref_name]
self.update_server_info()
return True
def create_branch(self, base_branch, new_branch, tracking=None):
"""Try creating a new branch which tracks the given remote
if such a branch does not exist then branch off a local branch
"""
# The remote to track
tracking = self.DEFAULT_REMOTE
# Already exists
if new_branch in self.branches:
raise Exception("branch %s already exists" % new_branch)
# Get information about remote_branch
remote_branch = os.path.sep.join([tracking, base_branch])
# Fork Local
if base_branch in self.branches:
base_ref = self._format_ref_branch(base_branch)
# Fork remote
elif remote_branch in self.remote_branches:
base_ref = self._format_ref_remote(remote_branch)
# TODO : track
else:
raise Exception(
"Can not find the branch named '%s' to fork either locally or in '%s'" % (base_branch, tracking)
)
# Reference of new branch
new_ref = self._format_ref_branch(new_branch)
# Copy reference to create branch
self.add_ref(new_ref, base_ref)
return new_ref
def remove_branch(self, branch_name):
ref = self._format_ref_branch(branch_name)
return self.remove_ref(ref)
def switch_branch(self, branch_name, tracking=None, create=None):
"""Changes the current branch
"""
if create is None:
create = True
# Check if branch exists
if not branch_name in self.branches:
self.create_branch(branch_name, branch_name, tracking=tracking)
# Get branch reference
branch_ref = self._format_ref_branch(branch_name)
# Change main branch
self.repo.refs.set_symbolic_ref("HEAD", branch_ref)
if self.is_working:
# Remove all files
self.clean_working()
# Add files for the current branch
self.checkout_all()
def clean(self, force=None, directories=None):
untracked_files = self.untracked_files
map(os.remove, untracked_files)
return untracked_files
def clean_working(self):
"""Purges all the working (removes everything except .git)
used by checkout_all to get clean branch switching
"""
return self.clean()
def _get_fs_structure(self, tree_sha, depth=None, parent_sha=None):
tree = self[tree_sha]
structure = {}
if depth is None:
depth = self.MAX_TREE_DEPTH
elif depth == 0:
return structure
for mode, path, sha in tree.entries():
# tree
if mode == self.MODE_DIRECTORY:
# Recur
structure[path] = self._get_fs_structure(sha, depth=depth - 1, parent_sha=tree_sha)
# commit
else:
structure[path] = sha
structure["."] = tree_sha
structure[".."] = parent_sha or tree_sha
return structure
def _get_fs_structure_by_path(self, tree_sha, path):
parts = path.split(os.path.sep)
depth = len(parts) + 1
structure = self._get_fs_structure(tree_sha, depth=depth)
return funky.subkey(structure, parts)
def commit_ls(self, ref, subpath=None):
"""List a "directory" for a given commit
using the tree of thqt commit
"""
tree_sha = self._commit_tree(ref)
# Root path
if subpath in self.ROOT_PATHS or not subpath:
return self._get_fs_structure(tree_sha, depth=1)
# Any other path
return self._get_fs_structure_by_path(tree_sha, subpath)
def commit_file(self, ref, path):
"""Return info on a given file for a given commit
"""
name, info = self.get_commit_files(ref, paths=[path]).items()[0]
return info
def commit_tree(self, ref, *args, **kwargs):
tree_sha = self._commit_tree(ref)
return self._get_fs_structure(tree_sha, *args, **kwargs)
def update_server_info(self):
if not self.is_bare:
return
update_server_info(self.repo)
def _is_fast_forward(self):
pass
def _merge_fast_forward(self):
pass
def __hash__(self):
"""This is required otherwise the memoize function will just mess it up
"""
return hash(self.path)
def __getitem__(self, key):
sha = self._parse_reference(key)
return self.repo[sha]
def __setitem__(self, key, value):
self.repo[key] = value
# Alias to clone_bare
fork = clone_bare
log = commit_info
diff_count = changes_count
comtributors = recent_contributors
class Gits3(object):
def __init__(self, path):
self.path = path
self.open_repo(path)
def open_repo(self, path):
self.repo = Repo(path)
def get_id(self, ref):
return self.repo.get_refs()[ref]
def get_updates(self, local_ref, tracking_ref):
refs = self.repo.get_refs()
for key, value in refs.iteritems():
print key, value
local = refs[local_ref]
try:
remote = refs[tracking_ref]
except KeyError:
remote = None
if local == remote:
return None
local_object = self.repo.get_object(local)
commits = self.get_commits(local_object, [remote])
objects = self.get_objects(commits)
print objects
if remote:
remote_object = self.repo.get_object(remote)
filtered_objects = self.filter_objects(objects, remote_object)
else:
filtered_objects = objects
filtered_objects = set(filtered_objects)
return filtered_objects
def filter_objects(self, objects, old_commit):
filtered = []
old_treeId = old_commit.tree
old_objects = self.get_objects_in_tree(old_treeId)
for object in objects:
if object not in old_objects:
filtered.append(object)
return filtered
def get_commits(self, interesting, uninteresting):
commits = [interesting]
remaining = interesting.parents
while remaining:
pId = remaining.pop(0)
if pId in uninteresting:
continue
else:
parent = self.repo.get_object(pId)
commits.append(parent)
parents = parent.parents
remaining.extend(parents)
return commits
def get_objects(self, commits):
objects = []
while commits:
commit = commits.pop(0)
objects.append(commit)
objects.extend(self.get_objects_in_tree(commit.tree))
return objects
def get_objects_in_tree(self, treeId):
objects = []
tree = self.repo.get_object(treeId)
objects.append(tree)
for entryId in tree.items():
# get the entry's sha
objectId = entryId[2]
object = self.repo.get_object(objectId)
if isinstance(object, Tree):
objects.extend(self.get_objects_in_tree(objectId))
else:
objects.append(object)
return objects
def generate_pack_name(self, objects):
m = hashlib.sha1()
for object in objects:
sha1 = object.sha().hexdigest()
# print sha1
m.update(sha1)
file_name = m.hexdigest()
# print 'File Name is ', file_name
return file_name
def write_pack(self, pack_name, objects):
write_pack('pack-' + pack_name, [(x, "") for x in objects],
len(objects))
def find_tracking_ref_names(self, fetch, refs):
if fetch[0] == '+':
fetch = fetch[1:]
tmp = fetch.split(':')
src = tmp[0]
dst = tmp[1]
# TODO double check that both src and dst have wild cards, or both don't
# match the source with refs
if src.endswith('*') and refs.startswith(src[:-1]):
return self.expand_from_src(src, dst, refs)
else:
return dst
def expand_from_src(self, src, dst, refs):
return dst[:-1] + refs[len(src) - 1:]
class DiskRefsContainerTests(RefsContainerTests, TestCase):
def setUp(self):
TestCase.setUp(self)
self._repo = open_repo('refs.git')
self.addCleanup(tear_down_repo, self._repo)
self._refs = self._repo.refs
def test_get_packed_refs(self):
self.assertEqual(
{
b'refs/heads/packed':
b'42d06bd4b77fed026b154d16493e5deab78f02ec',
b'refs/tags/refs-0.1':
b'df6800012397fb85c56e7418dd4eb9405dee075c',
}, self._refs.get_packed_refs())
def test_get_peeled_not_packed(self):
# not packed
self.assertEqual(None, self._refs.get_peeled(b'refs/tags/refs-0.2'))
self.assertEqual(b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
self._refs[b'refs/tags/refs-0.2'])
# packed, known not peelable
self.assertEqual(self._refs[b'refs/heads/packed'],
self._refs.get_peeled(b'refs/heads/packed'))
# packed, peeled
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.get_peeled(b'refs/tags/refs-0.1'))
def test_setitem(self):
RefsContainerTests.test_setitem(self)
f = open(os.path.join(self._refs.path, 'refs', 'some', 'ref'), 'rb')
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
f.read()[:40])
f.close()
def test_setitem_symbolic(self):
ones = b'1' * 40
self._refs[b'HEAD'] = ones
self.assertEqual(ones, self._refs[b'HEAD'])
# ensure HEAD was not modified
f = open(os.path.join(self._refs.path, 'HEAD'), 'rb')
v = next(iter(f)).rstrip(b'\n\r')
f.close()
self.assertEqual(b'ref: refs/heads/master', v)
# ensure the symbolic link was written through
f = open(os.path.join(self._refs.path, 'refs', 'heads', 'master'),
'rb')
self.assertEqual(ones, f.read()[:40])
f.close()
def test_set_if_equals(self):
RefsContainerTests.test_set_if_equals(self)
# ensure symref was followed
self.assertEqual(b'9' * 40, self._refs[b'refs/heads/master'])
# ensure lockfile was deleted
self.assertFalse(
os.path.exists(
os.path.join(self._refs.path, 'refs', 'heads', 'master.lock')))
self.assertFalse(
os.path.exists(os.path.join(self._refs.path, 'HEAD.lock')))
def test_add_if_new_packed(self):
# don't overwrite packed ref
self.assertFalse(
self._refs.add_if_new(b'refs/tags/refs-0.1', b'9' * 40))
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
def test_add_if_new_symbolic(self):
# Use an empty repo instead of the default.
repo_dir = os.path.join(tempfile.mkdtemp(), 'test')
os.makedirs(repo_dir)
repo = Repo.init(repo_dir)
self.addCleanup(tear_down_repo, repo)
refs = repo.refs
nines = b'9' * 40
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertFalse(b'refs/heads/master' in refs)
self.assertTrue(refs.add_if_new(b'HEAD', nines))
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
self.assertFalse(refs.add_if_new(b'HEAD', b'1' * 40))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
def test_follow(self):
self.assertEqual(([b'HEAD', b'refs/heads/master'
], b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs.follow(b'HEAD'))
self.assertEqual(([b'refs/heads/master'
], b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs.follow(b'refs/heads/master'))
self.assertRaises(KeyError, self._refs.follow, b'refs/heads/loop')
def test_delitem(self):
RefsContainerTests.test_delitem(self)
ref_file = os.path.join(self._refs.path, 'refs', 'heads', 'master')
self.assertFalse(os.path.exists(ref_file))
self.assertFalse(b'refs/heads/master' in self._refs.get_packed_refs())
def test_delitem_symbolic(self):
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
del self._refs[b'HEAD']
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs[b'refs/heads/master'])
self.assertFalse(os.path.exists(os.path.join(self._refs.path, 'HEAD')))
def test_remove_if_equals_symref(self):
# HEAD is a symref, so shouldn't equal its dereferenced value
self.assertFalse(
self._refs.remove_if_equals(
b'HEAD', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertTrue(
self._refs.remove_if_equals(
b'refs/heads/master',
b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/heads/master'])
# HEAD is now a broken symref
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
self.assertFalse(
os.path.exists(
os.path.join(self._refs.path, 'refs', 'heads', 'master.lock')))
self.assertFalse(
os.path.exists(os.path.join(self._refs.path, 'HEAD.lock')))
def test_remove_packed_without_peeled(self):
refs_file = os.path.join(self._repo.path, 'packed-refs')
f = GitFile(refs_file)
refs_data = f.read()
f.close()
f = GitFile(refs_file, 'wb')
f.write(b'\n'.join(l for l in refs_data.split(b'\n')
if not l or l[0] not in b'#^'))
f.close()
self._repo = Repo(self._repo.path)
refs = self._repo.refs
self.assertTrue(
refs.remove_if_equals(b'refs/heads/packed',
b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
def test_remove_if_equals_packed(self):
# test removing ref that is only packed
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
self.assertTrue(
self._refs.remove_if_equals(
b'refs/tags/refs-0.1',
b'df6800012397fb85c56e7418dd4eb9405dee075c'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/tags/refs-0.1'])
def test_read_ref(self):
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_ref(b'HEAD'))
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.read_ref(b'refs/heads/packed'))
self.assertEqual(None, self._refs.read_ref(b'nonexistant'))
def test_non_ascii(self):
try:
encoded_ref = u'refs/tags/schön'.encode(
sys.getfilesystemencoding())
except UnicodeEncodeError:
raise SkipTest(
"filesystem encoding doesn't support special character")
p = os.path.join(self._repo.path, 'refs', 'tags', u'schön')
with open(p, 'w') as f:
f.write('00' * 20)
expected_refs = dict(_TEST_REFS)
expected_refs[encoded_ref] = b'00' * 20
self.assertEqual(expected_refs, self._repo.get_refs())
class DiskRefsContainerTests(RefsContainerTests, TestCase):
def setUp(self):
TestCase.setUp(self)
self._repo = open_repo("refs.git")
self.addCleanup(tear_down_repo, self._repo)
self._refs = self._repo.refs
def test_get_packed_refs(self):
self.assertEqual(
{
b"refs/heads/packed":
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
b"refs/tags/refs-0.1":
b"df6800012397fb85c56e7418dd4eb9405dee075c",
},
self._refs.get_packed_refs(),
)
def test_get_peeled_not_packed(self):
# not packed
self.assertEqual(None, self._refs.get_peeled(b"refs/tags/refs-0.2"))
self.assertEqual(
b"3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8",
self._refs[b"refs/tags/refs-0.2"],
)
# packed, known not peelable
self.assertEqual(
self._refs[b"refs/heads/packed"],
self._refs.get_peeled(b"refs/heads/packed"),
)
# packed, peeled
self.assertEqual(
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
self._refs.get_peeled(b"refs/tags/refs-0.1"),
)
def test_setitem(self):
RefsContainerTests.test_setitem(self)
path = os.path.join(self._refs.path, b"refs", b"some", b"ref")
with open(path, "rb") as f:
self.assertEqual(b"42d06bd4b77fed026b154d16493e5deab78f02ec",
f.read()[:40])
self.assertRaises(
OSError,
self._refs.__setitem__,
b"refs/some/ref/sub",
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
)
def test_setitem_packed(self):
with open(os.path.join(self._refs.path, b"packed-refs"), "w") as f:
f.write("# pack-refs with: peeled fully-peeled sorted \n")
f.write(
"42d06bd4b77fed026b154d16493e5deab78f02ec refs/heads/packed\n")
# It's allowed to set a new ref on a packed ref, the new ref will be
# placed outside on refs/
self._refs[
b"refs/heads/packed"] = b"3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8"
packed_ref_path = os.path.join(self._refs.path, b"refs", b"heads",
b"packed")
with open(packed_ref_path, "rb") as f:
self.assertEqual(b"3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8",
f.read()[:40])
self.assertRaises(
OSError,
self._refs.__setitem__,
b"refs/heads/packed/sub",
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
)
def test_setitem_symbolic(self):
ones = b"1" * 40
self._refs[b"HEAD"] = ones
self.assertEqual(ones, self._refs[b"HEAD"])
# ensure HEAD was not modified
f = open(os.path.join(self._refs.path, b"HEAD"), "rb")
v = next(iter(f)).rstrip(b"\n\r")
f.close()
self.assertEqual(b"ref: refs/heads/master", v)
# ensure the symbolic link was written through
f = open(os.path.join(self._refs.path, b"refs", b"heads", b"master"),
"rb")
self.assertEqual(ones, f.read()[:40])
f.close()
def test_set_if_equals(self):
RefsContainerTests.test_set_if_equals(self)
# ensure symref was followed
self.assertEqual(b"9" * 40, self._refs[b"refs/heads/master"])
# ensure lockfile was deleted
self.assertFalse(
os.path.exists(
os.path.join(self._refs.path, b"refs", b"heads",
b"master.lock")))
self.assertFalse(
os.path.exists(os.path.join(self._refs.path, b"HEAD.lock")))
def test_add_if_new_packed(self):
# don't overwrite packed ref
self.assertFalse(
self._refs.add_if_new(b"refs/tags/refs-0.1", b"9" * 40))
self.assertEqual(
b"df6800012397fb85c56e7418dd4eb9405dee075c",
self._refs[b"refs/tags/refs-0.1"],
)
def test_add_if_new_symbolic(self):
# Use an empty repo instead of the default.
repo_dir = os.path.join(tempfile.mkdtemp(), "test")
os.makedirs(repo_dir)
repo = Repo.init(repo_dir)
self.addCleanup(tear_down_repo, repo)
refs = repo.refs
nines = b"9" * 40
self.assertEqual(b"ref: refs/heads/master", refs.read_ref(b"HEAD"))
self.assertFalse(b"refs/heads/master" in refs)
self.assertTrue(refs.add_if_new(b"HEAD", nines))
self.assertEqual(b"ref: refs/heads/master", refs.read_ref(b"HEAD"))
self.assertEqual(nines, refs[b"HEAD"])
self.assertEqual(nines, refs[b"refs/heads/master"])
self.assertFalse(refs.add_if_new(b"HEAD", b"1" * 40))
self.assertEqual(nines, refs[b"HEAD"])
self.assertEqual(nines, refs[b"refs/heads/master"])
def test_follow(self):
self.assertEqual(
(
[b"HEAD", b"refs/heads/master"],
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
),
self._refs.follow(b"HEAD"),
)
self.assertEqual(
(
[b"refs/heads/master"],
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
),
self._refs.follow(b"refs/heads/master"),
)
self.assertRaises(KeyError, self._refs.follow, b"refs/heads/loop")
def test_delitem(self):
RefsContainerTests.test_delitem(self)
ref_file = os.path.join(self._refs.path, b"refs", b"heads", b"master")
self.assertFalse(os.path.exists(ref_file))
self.assertFalse(b"refs/heads/master" in self._refs.get_packed_refs())
def test_delitem_symbolic(self):
self.assertEqual(b"ref: refs/heads/master",
self._refs.read_loose_ref(b"HEAD"))
del self._refs[b"HEAD"]
self.assertRaises(KeyError, lambda: self._refs[b"HEAD"])
self.assertEqual(
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
self._refs[b"refs/heads/master"],
)
self.assertFalse(os.path.exists(os.path.join(self._refs.path,
b"HEAD")))
def test_remove_if_equals_symref(self):
# HEAD is a symref, so shouldn't equal its dereferenced value
self.assertFalse(
self._refs.remove_if_equals(
b"HEAD", b"42d06bd4b77fed026b154d16493e5deab78f02ec"))
self.assertTrue(
self._refs.remove_if_equals(
b"refs/heads/master",
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
))
self.assertRaises(KeyError, lambda: self._refs[b"refs/heads/master"])
# HEAD is now a broken symref
self.assertRaises(KeyError, lambda: self._refs[b"HEAD"])
self.assertEqual(b"ref: refs/heads/master",
self._refs.read_loose_ref(b"HEAD"))
self.assertFalse(
os.path.exists(
os.path.join(self._refs.path, b"refs", b"heads",
b"master.lock")))
self.assertFalse(
os.path.exists(os.path.join(self._refs.path, b"HEAD.lock")))
def test_remove_packed_without_peeled(self):
refs_file = os.path.join(self._repo.path, "packed-refs")
f = GitFile(refs_file)
refs_data = f.read()
f.close()
f = GitFile(refs_file, "wb")
f.write(b"\n".join(line for line in refs_data.split(b"\n")
if not line or line[0] not in b"#^"))
f.close()
self._repo = Repo(self._repo.path)
refs = self._repo.refs
self.assertTrue(
refs.remove_if_equals(
b"refs/heads/packed",
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
))
def test_remove_if_equals_packed(self):
# test removing ref that is only packed
self.assertEqual(
b"df6800012397fb85c56e7418dd4eb9405dee075c",
self._refs[b"refs/tags/refs-0.1"],
)
self.assertTrue(
self._refs.remove_if_equals(
b"refs/tags/refs-0.1",
b"df6800012397fb85c56e7418dd4eb9405dee075c",
))
self.assertRaises(KeyError, lambda: self._refs[b"refs/tags/refs-0.1"])
def test_remove_parent(self):
self._refs[
b"refs/heads/foo/bar"] = b"df6800012397fb85c56e7418dd4eb9405dee075c"
del self._refs[b"refs/heads/foo/bar"]
ref_file = os.path.join(
self._refs.path,
b"refs",
b"heads",
b"foo",
b"bar",
)
self.assertFalse(os.path.exists(ref_file))
ref_file = os.path.join(self._refs.path, b"refs", b"heads", b"foo")
self.assertFalse(os.path.exists(ref_file))
ref_file = os.path.join(self._refs.path, b"refs", b"heads")
self.assertTrue(os.path.exists(ref_file))
self._refs[
b"refs/heads/foo"] = b"df6800012397fb85c56e7418dd4eb9405dee075c"
def test_read_ref(self):
self.assertEqual(b"ref: refs/heads/master",
self._refs.read_ref(b"HEAD"))
self.assertEqual(
b"42d06bd4b77fed026b154d16493e5deab78f02ec",
self._refs.read_ref(b"refs/heads/packed"),
)
self.assertEqual(None, self._refs.read_ref(b"nonexistant"))
def test_read_loose_ref(self):
self._refs[
b"refs/heads/foo"] = b"df6800012397fb85c56e7418dd4eb9405dee075c"
self.assertEqual(None, self._refs.read_ref(b"refs/heads/foo/bar"))
def test_non_ascii(self):
try:
encoded_ref = os.fsencode(u"refs/tags/schön")
except UnicodeEncodeError:
raise SkipTest(
"filesystem encoding doesn't support special character")
p = os.path.join(os.fsencode(self._repo.path), encoded_ref)
with open(p, "w") as f:
f.write("00" * 20)
expected_refs = dict(_TEST_REFS)
expected_refs[encoded_ref] = b"00" * 20
del expected_refs[b"refs/heads/loop"]
self.assertEqual(expected_refs, self._repo.get_refs())
def test_cyrillic(self):
if sys.platform in ("darwin", "win32"):
raise SkipTest(
"filesystem encoding doesn't support arbitrary bytes")
# reported in https://github.com/dulwich/dulwich/issues/608
name = b"\xcd\xee\xe2\xe0\xff\xe2\xe5\xf2\xea\xe01"
encoded_ref = b"refs/heads/" + name
with open(os.path.join(os.fsencode(self._repo.path), encoded_ref),
"w") as f:
f.write("00" * 20)
expected_refs = set(_TEST_REFS.keys())
expected_refs.add(encoded_ref)
self.assertEqual(expected_refs, set(self._repo.refs.allkeys()))
self.assertEqual(
{
r[len(b"refs/"):]
for r in expected_refs if r.startswith(b"refs/")
},
set(self._repo.refs.subkeys(b"refs/")),
)
expected_refs.remove(b"refs/heads/loop")
expected_refs.add(b"HEAD")
self.assertEqual(expected_refs, set(self._repo.get_refs().keys()))
class Game(object):
"A versioned game"
def __init__(self, name=None, **options):
self.name = name or uuid.uuid4().hex
self.options = dict(DEFAULTS, **options)
self.data = self.options.pop('data').format(name=self.name)
new = False
self.repo = None
if not os.path.exists(self.data):
if not self.options['create']: raise GameError("Game does not exist")
os.makedirs(self.data)
try:
self.repo = Repo(self.data)
except dulwich.errors.NotGitRepository:
if not self.options['create']: raise GameError("Game does not exist")
self.repo = Repo.init_bare(self.data)
new = True
self.board = (new and BoardState()) or self.get_board()
if new: self.save("New blank board for game: %s" % self.name)
@property
def branch(self):
head = self.repo.refs.read_ref('HEAD')
if head and head.startswith('ref: '):
head = head.split(': ')[-1]
head = head.replace('refs/heads/', '')
return head
return 'master'
def _tree(self, branch=None):
branch = branch or self.branch
try: return self.repo[
self.repo['refs/heads/%s' % branch].tree
]
except KeyError: return Tree()
def signature(self, of=None):
of = (of and "refs/heads/%s" % of) or "HEAD"
try: return self.repo.refs[of]
except KeyError: return None
def get_board(self, branch=None):
branch = branch or self.branch
if branch not in self.branches(): raise GameError("Unknown branch")
return BoardState.from_json(
self.repo[
[t[2]
for t in self._tree(branch).entries() # [(mode, name, sha)...]
if t[1] == 'board.json'].pop()
].data)
def set_branch(self, new):
if 'refs/heads/%s' % new in self.repo.get_refs().keys():
self.repo.refs.set_symbolic_ref('HEAD', 'refs/heads/%s' % new)
return self.branch
return False
def branches(self):
return sorted([name.replace('refs/heads/', '')
for (name, sig) in self.repo.get_refs().items()
if name != "HEAD"])
def make_branch(self, name, back=0):
if ('refs/heads/%s' % name) in self.repo.get_refs().keys():
raise GameError("I already have this branch")
try:
head = self.repo.head()
history = self.repo.revision_history(head)
self.repo.refs['refs/heads/%s' % name] = history[back].id
except IndexError:
raise GameError("Trying to go {back} which is further than history".format(back=back))
return True
def save(self, message="Forced commit"):
blob = Blob.from_string(self.board.as_json())
tree = self._tree()
tree.add(0100644, 'board.json', blob.id)
[self.repo.object_store.add_object(it)
for it in (blob, tree)]
self.repo.do_commit(message, committer="Game %s" % self.name, tree=tree.id)
def move(self, x, y):
player = self.board.player_turn()
if not self.board.game_over and self.board.move(x, y):
self.save("{player} moved to ({x}, {y})".format(player=player,
x=x,
y=y))
return player
return None
def skip(self):
player = self.board.player_turn()
if not self.board.game_over:
self.board.move(None)
is_or_isnt = (self.board.game_over and "is") or "is NOT"
self.save("{player} skipped, game {maybe} over".format(player=player,
maybe=is_or_isnt))
return self.board.game_over or self.board.player_turn()
def who(self):
return self.board.player_turn()
def scores(self):
return self.board.scores()
def winner(self):
return self.board.winner
def __unicode__(self):
return "Game: {name} {black} vs {white} on {x}x{y} from {data} :: {board}".format(
name=self.name,
board=self.board,
data=self.data,
**self.options
)
__str__=__unicode__
def __repr__(self):
return "<%s>" % self
class DiskRefsContainerTests(RefsContainerTests, TestCase):
def setUp(self):
TestCase.setUp(self)
self._repo = open_repo('refs.git')
self.addCleanup(tear_down_repo, self._repo)
self._refs = self._repo.refs
def test_get_packed_refs(self):
self.assertEqual({
b'refs/heads/packed': b'42d06bd4b77fed026b154d16493e5deab78f02ec',
b'refs/tags/refs-0.1': b'df6800012397fb85c56e7418dd4eb9405dee075c',
}, self._refs.get_packed_refs())
def test_get_peeled_not_packed(self):
# not packed
self.assertEqual(None, self._refs.get_peeled(b'refs/tags/refs-0.2'))
self.assertEqual(b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8',
self._refs[b'refs/tags/refs-0.2'])
# packed, known not peelable
self.assertEqual(self._refs[b'refs/heads/packed'],
self._refs.get_peeled(b'refs/heads/packed'))
# packed, peeled
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.get_peeled(b'refs/tags/refs-0.1'))
def test_setitem(self):
RefsContainerTests.test_setitem(self)
f = open(os.path.join(self._refs.path, 'refs', 'some', 'ref'), 'rb')
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
f.read()[:40])
f.close()
def test_setitem_symbolic(self):
ones = b'1' * 40
self._refs[b'HEAD'] = ones
self.assertEqual(ones, self._refs[b'HEAD'])
# ensure HEAD was not modified
f = open(os.path.join(self._refs.path, 'HEAD'), 'rb')
self.assertEqual(b'ref: refs/heads/master', next(iter(f)).rstrip(b'\n'))
f.close()
# ensure the symbolic link was written through
f = open(os.path.join(self._refs.path, 'refs', 'heads', 'master'), 'rb')
self.assertEqual(ones, f.read()[:40])
f.close()
def test_set_if_equals(self):
RefsContainerTests.test_set_if_equals(self)
# ensure symref was followed
self.assertEqual(b'9' * 40, self._refs[b'refs/heads/master'])
# ensure lockfile was deleted
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, 'refs', 'heads', 'master.lock')))
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, 'HEAD.lock')))
def test_add_if_new_packed(self):
# don't overwrite packed ref
self.assertFalse(self._refs.add_if_new(b'refs/tags/refs-0.1',
b'9' * 40))
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
def test_add_if_new_symbolic(self):
# Use an empty repo instead of the default.
repo_dir = os.path.join(tempfile.mkdtemp(), 'test')
os.makedirs(repo_dir)
repo = Repo.init(repo_dir)
self.addCleanup(tear_down_repo, repo)
refs = repo.refs
nines = b'9' * 40
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertFalse(b'refs/heads/master' in refs)
self.assertTrue(refs.add_if_new(b'HEAD', nines))
self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD'))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
self.assertFalse(refs.add_if_new(b'HEAD', b'1' * 40))
self.assertEqual(nines, refs[b'HEAD'])
self.assertEqual(nines, refs[b'refs/heads/master'])
def test_follow(self):
self.assertEqual((b'refs/heads/master',
b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs._follow(b'HEAD'))
self.assertEqual((b'refs/heads/master',
b'42d06bd4b77fed026b154d16493e5deab78f02ec'),
self._refs._follow(b'refs/heads/master'))
self.assertRaises(KeyError, self._refs._follow, b'refs/heads/loop')
def test_delitem(self):
RefsContainerTests.test_delitem(self)
ref_file = os.path.join(self._refs.path, 'refs', 'heads', 'master')
self.assertFalse(os.path.exists(ref_file))
self.assertFalse(b'refs/heads/master' in self._refs.get_packed_refs())
def test_delitem_symbolic(self):
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
del self._refs[b'HEAD']
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs[b'refs/heads/master'])
self.assertFalse(os.path.exists(os.path.join(self._refs.path, 'HEAD')))
def test_remove_if_equals_symref(self):
# HEAD is a symref, so shouldn't equal its dereferenced value
self.assertFalse(self._refs.remove_if_equals(
b'HEAD', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertTrue(self._refs.remove_if_equals(
b'refs/heads/master', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/heads/master'])
# HEAD is now a broken symref
self.assertRaises(KeyError, lambda: self._refs[b'HEAD'])
self.assertEqual(b'ref: refs/heads/master',
self._refs.read_loose_ref(b'HEAD'))
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, 'refs', 'heads', 'master.lock')))
self.assertFalse(os.path.exists(
os.path.join(self._refs.path, 'HEAD.lock')))
def test_remove_packed_without_peeled(self):
refs_file = os.path.join(self._repo.path, 'packed-refs')
f = GitFile(refs_file)
refs_data = f.read()
f.close()
f = GitFile(refs_file, 'wb')
f.write(b'\n'.join(l for l in refs_data.split(b'\n')
if not l or l[0] not in b'#^'))
f.close()
self._repo = Repo(self._repo.path)
refs = self._repo.refs
self.assertTrue(refs.remove_if_equals(
b'refs/heads/packed', b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
def test_remove_if_equals_packed(self):
# test removing ref that is only packed
self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c',
self._refs[b'refs/tags/refs-0.1'])
self.assertTrue(
self._refs.remove_if_equals(
b'refs/tags/refs-0.1',
b'df6800012397fb85c56e7418dd4eb9405dee075c'))
self.assertRaises(KeyError, lambda: self._refs[b'refs/tags/refs-0.1'])
def test_read_ref(self):
self.assertEqual(b'ref: refs/heads/master', self._refs.read_ref(b'HEAD'))
self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec',
self._refs.read_ref(b'refs/heads/packed'))
self.assertEqual(None, self._refs.read_ref(b'nonexistant'))
def test_non_ascii(self):
try:
encoded_ref = u'refs/tags/schön'.encode(sys.getfilesystemencoding())
except UnicodeEncodeError:
raise SkipTest("filesystem encoding doesn't support special character")
p = os.path.join(self._repo.path, 'refs', 'tags', 'schön')
with open(p, 'w') as f:
f.write('00' * 20)
expected_refs = dict(_TEST_REFS)
expected_refs[encoded_ref] = b'00' * 20
self.assertEqual(expected_refs, self._repo.get_refs())
class GitHandler(object):
mapfile = 'git-mapfile'
tagsfile = 'git-tags'
def __init__(self, dest_repo, ui):
self.repo = dest_repo
self.ui = ui
if ui.configbool('git', 'intree'):
self.gitdir = self.repo.wjoin('.git')
else:
self.gitdir = self.repo.join('git')
self.paths = ui.configitems('paths')
self.load_map()
self.load_tags()
# make the git data directory
def init_if_missing(self):
if os.path.exists(self.gitdir):
self.git = Repo(self.gitdir)
else:
os.mkdir(self.gitdir)
self.git = Repo.init_bare(self.gitdir)
## FILE LOAD AND SAVE METHODS
def map_set(self, gitsha, hgsha):
self._map_git[gitsha] = hgsha
self._map_hg[hgsha] = gitsha
def map_hg_get(self, gitsha):
return self._map_git.get(gitsha)
def map_git_get(self, hgsha):
return self._map_hg.get(hgsha)
def load_map(self):
self._map_git = {}
self._map_hg = {}
if os.path.exists(self.repo.join(self.mapfile)):
for line in self.repo.opener(self.mapfile):
gitsha, hgsha = line.strip().split(' ', 1)
self._map_git[gitsha] = hgsha
self._map_hg[hgsha] = gitsha
def save_map(self):
file = self.repo.opener(self.mapfile, 'w+', atomictemp=True)
for hgsha, gitsha in sorted(self._map_hg.iteritems()):
file.write("%s %s\n" % (gitsha, hgsha))
# If this complains that NoneType is not callable, then
# atomictempfile no longer has either of rename (pre-1.9) or
# close (post-1.9)
getattr(file, 'rename', getattr(file, 'close', None))()
def load_tags(self):
self.tags = {}
if os.path.exists(self.repo.join(self.tagsfile)):
for line in self.repo.opener(self.tagsfile):
sha, name = line.strip().split(' ', 1)
self.tags[name] = sha
def save_tags(self):
file = self.repo.opener(self.tagsfile, 'w+', atomictemp=True)
for name, sha in sorted(self.tags.iteritems()):
if not self.repo.tagtype(name) == 'global':
file.write("%s %s\n" % (sha, name))
# If this complains that NoneType is not callable, then
# atomictempfile no longer has either of rename (pre-1.9) or
# close (post-1.9)
getattr(file, 'rename', getattr(file, 'close', None))()
## END FILE LOAD AND SAVE METHODS
## COMMANDS METHODS
def import_commits(self, remote_name):
self.import_git_objects(remote_name)
self.update_hg_bookmarks(self.git.get_refs())
self.save_map()
def fetch(self, remote, heads):
self.export_commits()
refs = self.fetch_pack(remote, heads)
remote_name = self.remote_name(remote)
oldrefs = self.git.get_refs()
if refs:
self.import_git_objects(remote_name, refs)
self.import_tags(refs)
self.update_hg_bookmarks(refs)
if remote_name:
self.update_remote_branches(remote_name, refs)
elif not self.paths:
# intial cloning
self.update_remote_branches('default', refs)
# "Activate" a tipmost bookmark.
bms = getattr(self.repo['tip'], 'bookmarks',
lambda : None)()
if bms:
bookmarks.setcurrent(self.repo, bms[0])
def remoteref(ref):
rn = remote_name or 'default'
return 'refs/remotes/' + rn + ref[10:]
modheads = [refs[k] for k in refs if k.startswith('refs/heads/')
and not k.endswith('^{}')
and refs[k] != oldrefs.get(remoteref(k))]
if not modheads:
self.ui.status(_("no changes found\n"))
self.save_map()
return len(modheads)
def export_commits(self):
try:
self.export_git_objects()
self.export_hg_tags()
self.update_references()
finally:
self.save_map()
def get_refs(self, remote):
self.export_commits()
client, path = self.get_transport_and_path(remote)
old_refs = {}
new_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = set(self.local_heads().values() + self.tags.values())
new_refs.update(self.get_changed_refs(refs, to_push, True))
# don't push anything
return {}
try:
client.send_pack(path, changed, None)
changed_refs = [ref for ref, sha in new_refs.iteritems()
if sha != old_refs.get(ref)]
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
old = {}
for r in old_refs:
old_ref = self.map_hg_get(old_refs[r])
if old_ref:
old[bin(old_ref)] = 1
return old, new
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
def cli(metadata_template):
"""Generate the 'metadata.json' file for this app."""
metadata_dir = Path(__file__).resolve().parent
output_dir = metadata_dir / 'build'
root = metadata_dir.parent
repo = Repo(root)
versions = {
parse(get_version_identifier(ref.decode())): ref
for ref in repo.get_refs()
}
requirements = {
version: get_requirements_for_ref(repo, ref)
for version, ref in versions.items()
}
def get_requirements(spec, version=None):
spec = SpecifierSet(spec)
if version is None:
matching_versions = [
version for version in sorted(versions) if version in spec
]
matching_requirements = {
requirements[version]
for version in matching_versions
}
if len(matching_requirements) == 0:
raise RuntimeError(
f"Unable to determine requirements for specifier '{spec}'."
)
elif len(matching_requirements) > 1:
raise RuntimeError(
f"Requirements for specifier '{spec}' are not uniform.")
reqs = matching_requirements.pop()
else:
reqs = requirements[parse(version)]
return json.dumps({str(spec): reqs})[1:-1]
env = Environment(loader=FileSystemLoader(metadata_dir / 'templates'))
env.filters['get_requirements'] = get_requirements
# Writing output...
output_dir.mkdir(exist_ok=True)
# index.html
index_html = output_dir / 'index.html'
index_html.write_text(env.get_template('index.html').render())
click.echo(f"Write {index_html.relative_to(Path.cwd())}")
# metadata.json
metadata_json = output_dir / 'metadata.json'
metadata_template = env.get_template(metadata_template)
try:
metadata = json.loads(metadata_template.render())
except json.decoder.JSONDecodeError as error:
raise RuntimeError(f"{error}\n{metadata_template.render()}")
validate(
instance=metadata,
schema={
"$ref":
"https://aiidalab.github.io/aiidalab-registry/schemas/v1/metadata.schema.json"
})
metadata_json.write_text(json.dumps(metadata, indent=2))
click.echo(f"Write {metadata_json.relative_to(Path.cwd())}")
# QE.jpg
src = root / 'miscellaneous' / 'logos' / 'QE.jpg'
dst = output_dir / 'miscellaneous' / 'logos' / 'QE.jpg'
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(src, dst)
click.echo(f"Copy {dst.relative_to(Path.cwd())}")
class GitHandler(object):
mapfile = 'git-mapfile'
tagsfile = 'git-tags'
def __init__(self, dest_repo, ui):
self.repo = dest_repo
self.ui = ui
if ui.configbool('git', 'intree'):
self.gitdir = self.repo.wjoin('.git')
else:
self.gitdir = self.repo.join('git')
self.paths = ui.configitems('paths')
self.load_map()
self.load_tags()
# make the git data directory
def init_if_missing(self):
if os.path.exists(self.gitdir):
self.git = Repo(self.gitdir)
else:
os.mkdir(self.gitdir)
self.git = Repo.init_bare(self.gitdir)
## FILE LOAD AND SAVE METHODS
def map_set(self, gitsha, hgsha):
self._map_git[gitsha] = hgsha
self._map_hg[hgsha] = gitsha
def map_hg_get(self, gitsha):
return self._map_git.get(gitsha)
def map_git_get(self, hgsha):
return self._map_hg.get(hgsha)
def load_map(self):
self._map_git = {}
self._map_hg = {}
if os.path.exists(self.repo.join(self.mapfile)):
for line in self.repo.opener(self.mapfile):
gitsha, hgsha = line.strip().split(' ', 1)
self._map_git[gitsha] = hgsha
self._map_hg[hgsha] = gitsha
def save_map(self):
file = self.repo.opener(self.mapfile, 'w+', atomictemp=True)
for hgsha, gitsha in sorted(self._map_hg.iteritems()):
file.write("%s %s\n" % (gitsha, hgsha))
file.rename()
def load_tags(self):
self.tags = {}
if os.path.exists(self.repo.join(self.tagsfile)):
for line in self.repo.opener(self.tagsfile):
sha, name = line.strip().split(' ', 1)
self.tags[name] = sha
def save_tags(self):
file = self.repo.opener(self.tagsfile, 'w+', atomictemp=True)
for name, sha in sorted(self.tags.iteritems()):
if not self.repo.tagtype(name) == 'global':
file.write("%s %s\n" % (sha, name))
file.rename()
## END FILE LOAD AND SAVE METHODS
## COMMANDS METHODS
def import_commits(self, remote_name):
self.import_git_objects(remote_name)
self.save_map()
def fetch(self, remote, heads):
self.export_commits()
refs = self.fetch_pack(remote, heads)
remote_name = self.remote_name(remote)
oldrefs = self.git.get_refs()
if refs:
self.import_git_objects(remote_name, refs)
self.import_tags(refs)
self.update_hg_bookmarks(refs)
if remote_name:
self.update_remote_branches(remote_name, refs)
elif not self.paths:
# intial cloning
self.update_remote_branches('default', refs)
# "Activate" a tipmost bookmark.
bms = getattr(self.repo['tip'], 'bookmarks',
lambda : None)()
if bms:
bookmarks.setcurrent(self.repo, bms[0])
def remoteref(ref):
rn = remote_name or 'default'
return 'refs/remotes/' + rn + ref[10:]
modheads = [refs[k] for k in refs if k.startswith('refs/heads/')
and not k.endswith('^{}')
and refs[k] != oldrefs.get(remoteref(k))]
if not modheads:
self.ui.status(_("no changes found\n"))
self.save_map()
return len(modheads)
def export_commits(self):
try:
self.export_git_objects()
self.export_hg_tags()
self.update_references()
finally:
self.save_map()
def get_refs(self, remote):
self.export_commits()
client, path = self.get_transport_and_path(remote)
old_refs = {}
new_refs = {}
def changed(refs):
old_refs.update(refs)
to_push = set(self.local_heads().values() + self.tags.values())
new_refs.update(self.get_changed_refs(refs, to_push, True))
# don't push anything
return {}
try:
client.send_pack(path, changed, None)
changed_refs = [ref for ref, sha in new_refs.iteritems()
if sha != old_refs.get(ref)]
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
old = dict( (bin(self.map_hg_get(old_refs[r])), 1)
for r in old_refs)
return old, new
except (HangupException, GitProtocolError), e:
raise hgutil.Abort(_("git remote error: ") + str(e))
def get_branches(repo_path):
repo = Repo(repo_path)
return [ref[11:] for ref in repo.get_refs()
if ref.startswith('refs/heads/')]
class Gittle(object):
"""All paths used in Gittle external methods must be paths relative to the git repository
"""
DEFAULT_COMMIT = 'HEAD'
DEFAULT_BRANCH = 'master'
DEFAULT_REMOTE = 'origin'
DEFAULT_MESSAGE = '**No Message**'
DEFAULT_USER_INFO = {
'name': None,
'email': None,
}
DIFF_FUNCTIONS = {
'classic': utils.git.classic_tree_diff,
'dict': utils.git.dict_tree_diff,
'changes': utils.git.dict_tree_diff
}
DEFAULT_DIFF_TYPE = 'dict'
HIDDEN_REGEXES = [
# Hide git directory
r'.*\/\.git\/.*',
]
# References
REFS_BRANCHES = 'refs/heads/'
REFS_REMOTES = 'refs/remotes/'
REFS_TAGS = 'refs/tags/'
# Name pattern truths
# Used for detecting if files are :
# - deleted
# - added
# - changed
PATTERN_ADDED = (False, True)
PATTERN_REMOVED = (True, False)
PATTERN_MODIFIED = (True, True)
# Permissions
MODE_DIRECTORY = 0o40000 # Used to tell if a tree entry is a directory
# Tree depth
MAX_TREE_DEPTH = 1000
# Acceptable Root paths
ROOT_PATHS = (os.path.curdir, os.path.sep)
def __init__(self, repo_or_path, origin_uri=None, auth=None, report_activity=None, *args, **kwargs):
if isinstance(repo_or_path, DulwichRepo):
self.repo = repo_or_path
elif isinstance(repo_or_path, Gittle):
self.repo = DulwichRepo(repo_or_path.path)
elif isinstance(repo_or_path, basestring):
path = os.path.abspath(repo_or_path)
self.repo = DulwichRepo(path)
else:
logging.warning('Repo is of type %s' % type(repo_or_path))
raise Exception('Gittle must be initialized with either a dulwich repository or a string to the path')
# Set path
self.path = self.repo.path
# The remote url
self.origin_uri = origin_uri
# Report client activty
self._report_activity = report_activity
# Build ignore filter
self.hidden_regexes = copy.copy(self.HIDDEN_REGEXES)
self.hidden_regexes.extend(self._get_ignore_regexes())
self.ignore_filter = utils.paths.path_filter_regex(self.hidden_regexes)
self.filters = [
self.ignore_filter,
]
# Get authenticator
if auth:
self.authenticator = auth
else:
self.auth(*args, **kwargs)
def report_activity(self, *args, **kwargs):
if not self._report_activity:
return
return self._report_activity(*args, **kwargs)
def _format_author(self, name, email):
return "%s <%s>" % (name, email)
def _format_userinfo(self, userinfo):
name = userinfo.get('name')
email = userinfo.get('email')
if name and email:
return self._format_author(name, email)
return None
def _format_ref(self, base, extra):
return ''.join([base, extra])
def _format_ref_branch(self, branch_name):
return self._format_ref(self.REFS_BRANCHES, branch_name)
def _format_ref_remote(self, remote_name):
return self._format_ref(self.REFS_REMOTES, remote_name)
def _format_ref_tag(self, tag_name):
return self._format_ref(self.REFS_TAGS, tag_name)
@property
def head(self):
"""Return SHA of the current HEAD
"""
return self.repo.head()
@property
def is_bare(self):
"""Bare repositories have no working directories or indexes
"""
return self.repo.bare
@property
def is_working(self):
return not(self.is_bare)
def has_index(self):
"""Opposite of is_bare
"""
return self.repo.has_index()
@property
def has_commits(self):
"""
If the repository has no HEAD we consider that is has no commits
"""
try:
self.repo.head()
except KeyError:
return False
return True
def ref_walker(self, ref=None):
"""
Very simple, basic walker
"""
ref = ref or 'HEAD'
sha = self._commit_sha(ref)
for entry in self.repo.get_walker(sha):
yield entry.commit
def branch_walker(self, branch):
branch = branch or self.active_branch
ref = self._format_ref_branch(branch)
return self.ref_walker(ref)
def commit_info(self, start=0, end=None, branch=None):
"""Return a generator of commits with all their attached information
"""
if not self.has_commits:
return []
commits = [utils.git.commit_info(entry) for entry in self.branch_walker(branch)]
if not end:
return commits
return commits[start:end]
@funky.uniquify
def recent_contributors(self, n=None, branch=None):
n = n or 10
return funky.pluck(self.commit_info(end=n, branch=branch), 'author')
@property
def commit_count(self):
try:
return len(self.ref_walker())
except KeyError:
return 0
def commits(self):
"""Return a list of SHAs for all the concerned commits
"""
return [commit['sha'] for commit in self.commit_info()]
@property
def git_dir(self):
return self.repo.controldir()
def auth(self, *args, **kwargs):
self.authenticator = GittleAuth(*args, **kwargs)
return self.authenticator
# Generate a branch selector (used for pushing)
def _wants_branch(self, branch_name=None):
branch_name = branch_name or self.active_branch
refs_key = self._format_ref_branch(branch_name)
sha = self.branches[branch_name]
def wants_func(old):
refs_key = self._format_ref_branch(branch_name)
return {
refs_key: sha
}
return wants_func
def _get_ignore_regexes(self):
gitignore_filename = os.path.join(self.path, '.gitignore')
if not os.path.exists(gitignore_filename):
return []
lines = open(gitignore_filename).readlines()
globers = map(lambda line: line.rstrip(), lines)
return utils.paths.globers_to_regex(globers)
# Get the absolute path for a file in the git repo
def abspath(self, repo_file):
return os.path.abspath(
os.path.join(self.path, repo_file)
)
# Get the relative path from the absolute path
def relpath(self, abspath):
return os.path.relpath(abspath, self.path)
@property
def last_commit(self):
return self[self.repo.head()]
@property
def index(self):
return self.repo.open_index()
@classmethod
def init(cls, path, bare=None, *args, **kwargs):
"""Initialize a repository"""
mkdir_safe(path)
# Constructor to use
if bare:
constructor = DulwichRepo.init_bare
else:
constructor = DulwichRepo.init
# Create dulwich repo
repo = constructor(path)
# Create Gittle repo
return cls(repo, *args, **kwargs)
@classmethod
def init_bare(cls, *args, **kwargs):
kwargs.setdefault('bare', True)
return cls.init(*args, **kwargs)
@classmethod
def is_repo(cls, path):
"""Returns True if path is a git repository, False if it is not"""
try:
repo = Gittle(path)
except NotGitRepository:
return False
else:
return True
def get_client(self, origin_uri=None, **kwargs):
# Get the remote URL
origin_uri = origin_uri or self.origin_uri
# Fail if inexistant
if not origin_uri:
raise InvalidRemoteUrl()
client_kwargs = {}
auth_kwargs = self.authenticator.kwargs()
client_kwargs.update(auth_kwargs)
client_kwargs.update(kwargs)
client_kwargs.update({
'report_activity': self.report_activity
})
client, remote_path = get_transport_and_path(origin_uri, **client_kwargs)
return client, remote_path
def push_to(self, origin_uri, branch_name=None, progress=None):
selector = self._wants_branch(branch_name=branch_name)
client, remote_path = self.get_client(origin_uri)
return client.send_pack(
remote_path,
selector,
self.repo.object_store.generate_pack_contents,
progress=progress
)
# Like: git push
def push(self, origin_uri=None, branch_name=None, progress=None):
return self.push_to(origin_uri, branch_name, progress)
# Not recommended at ALL ... !!!
def dirty_pull_from(self, origin_uri, branch_name=None):
# Remove all previously existing data
rmtree(self.path)
mkdir_safe(self.path)
self.repo = DulwichRepo.init(self.path)
# Fetch brand new copy from remote
return self.pull_from(origin_uri, branch_name)
def pull_from(self, origin_uri, branch_name=None):
return self.fetch(origin_uri)
# Like: git pull
def pull(self, origin_uri=None, branch_name=None):
return self.pull_from(origin_uri, branch_name)
def fetch_remote(self, origin_uri=None):
# Get client
client, remote_path = self.get_client(origin_uri=origin_uri)
# Fetch data from remote repository
remote_refs = client.fetch(remote_path, self.repo)
return remote_refs
def _setup_fetched_refs(self, refs, origin, bare):
remote_tags = utils.git.subrefs(refs, 'refs/tags')
remote_heads = utils.git.subrefs(refs, 'refs/heads')
# Filter refs
clean_remote_tags = utils.git.clean_refs(remote_tags)
clean_remote_heads = utils.git.clean_refs(remote_heads)
# Base of new refs
heads_base = 'refs/remotes/' + origin
if bare:
heads_base = 'refs/heads'
# Import branches
self.import_refs(
heads_base,
clean_remote_heads
)
# Import tags
self.import_refs(
'refs/tags',
clean_remote_tags
)
# Update HEAD
for k, v in utils.git.clean_refs(refs).items():
self[k] = v
def fetch(self, origin_uri=None, bare=None, origin=None):
bare = bare or False
origin = origin or self.DEFAULT_REMOTE
# Remote refs
remote_refs = self.fetch_remote(origin_uri)
# Update head
# Hit repo because head doesn't yet exist so
# print("REFS = %s" % remote_refs)
# If no refs (empty repository()
if not remote_refs:
return
# Update refs (branches, tags, HEAD)
self._setup_fetched_refs(remote_refs, origin, bare)
# Checkout working directories
if not bare and self.has_commits:
self.checkout_all()
else:
self.update_server_info()
@classmethod
def clone(cls, origin_uri, local_path, auth=None, mkdir=True, bare=False, *args, **kwargs):
"""Clone a remote repository"""
mkdir_safe(local_path)
# Initialize the local repository
if bare:
local_repo = cls.init_bare(local_path)
else:
local_repo = cls.init(local_path)
repo = cls(local_repo, origin_uri=origin_uri, auth=auth, *args, **kwargs)
repo.fetch(bare=bare)
# Add origin
repo.add_remote('origin', origin_uri)
return repo
@classmethod
def clone_bare(cls, *args, **kwargs):
"""Same as .clone except clones to a bare repository by default
"""
kwargs.setdefault('bare', True)
return cls.clone(*args, **kwargs)
def _commit(self, committer=None, author=None, message=None, files=None, tree=None, *args, **kwargs):
if not tree:
# If no tree then stage files
modified_files = files or self.modified_files
logging.info("STAGING : %s" % modified_files)
self.repo.stage(modified_files)
# Messages
message = message or self.DEFAULT_MESSAGE
author_msg = self._format_userinfo(author)
committer_msg = self._format_userinfo(committer)
return self.repo.do_commit(
message=message,
author=author_msg,
committer=committer_msg,
encoding='UTF-8',
tree=tree,
*args, **kwargs
)
def _tree_from_structure(self, structure):
# TODO : Support directories
tree = Tree()
for file_info in structure:
# str only
try:
data = file_info['data'].encode('ascii')
name = file_info['name'].encode('ascii')
mode = file_info['mode']
except:
# Skip file on encoding errors
continue
blob = Blob()
blob.data = data
# Store file's contents
self.repo.object_store.add_object(blob)
# Add blob entry
tree.add(
name,
mode,
blob.id
)
# Store tree
self.repo.object_store.add_object(tree)
return tree.id
# Like: git commmit -a
def commit(self, name=None, email=None, message=None, files=None, *args, **kwargs):
user_info = {
'name': name,
'email': email,
}
return self._commit(
committer=user_info,
author=user_info,
message=message,
files=files,
*args,
**kwargs
)
def commit_structure(self, name=None, email=None, message=None, structure=None, *args, **kwargs):
"""Main use is to do commits directly to bare repositories
For example doing a first Initial Commit so the repo can be cloned and worked on right away
"""
if not structure:
return
tree = self._tree_from_structure(structure)
user_info = {
'name': name,
'email': email,
}
return self._commit(
committer=user_info,
author=user_info,
message=message,
tree=tree,
*args,
**kwargs
)
# Push all local commits
# and pull all remote commits
def sync(self, origin_uri=None):
self.push(origin_uri)
return self.pull(origin_uri)
def lookup_entry(self, relpath, trackable_files=set()):
if not relpath in trackable_files:
raise KeyError
abspath = self.abspath(relpath)
with open(abspath, 'rb') as git_file:
data = git_file.read()
s = sha1()
s.update("blob %u\0" % len(data))
s.update(data)
return (s.hexdigest(), os.stat(abspath).st_mode)
@property
@funky.transform(set)
def tracked_files(self):
return list(self.index)
@property
@funky.transform(set)
def raw_files(self):
return utils.paths.subpaths(self.path)
@property
@funky.transform(set)
def ignored_files(self):
return utils.paths.subpaths(self.path, filters=self.filters)
@property
@funky.transform(set)
def trackable_files(self):
return self.raw_files - self.ignored_files
@property
@funky.transform(set)
def untracked_files(self):
return self.trackable_files - self.tracked_files
"""
@property
@funky.transform(set)
def modified_staged_files(self):
"Checks if the file has changed since last commit"
timestamp = self.last_commit.commit_time
index = self.index
return [
f
for f in self.tracked_files
if index[f][1][0] > timestamp
]
"""
# Return a list of tuples
# representing the changed elements in the git tree
def _changed_entries(self, ref=None):
ref = ref or self.DEFAULT_COMMIT
if not self.has_commits:
return []
obj_sto = self.repo.object_store
tree_id = self[ref].tree
names = self.trackable_files
lookup_func = partial(self.lookup_entry, trackable_files=names)
# Format = [((old_name, new_name), (old_mode, new_mode), (old_sha, new_sha)), ...]
tree_diff = changes_from_tree(names, lookup_func, obj_sto, tree_id, want_unchanged=False)
return list(tree_diff)
@funky.transform(set)
def _changed_entries_by_pattern(self, pattern):
changed_entries = self._changed_entries()
filtered_paths = None
#if the pattern is PATTERN_MODIFIED, should check the sha
if self.PATTERN_MODIFIED == pattern:
filtered_paths = [
funky.first_true(names)
for names, modes, sha in changed_entries
if tuple(map(bool, names)) == pattern and funky.first_true(names) and sha[0] == sha[1]
]
else :
filtered_paths = [
funky.first_true(names)
for names, modes, sha in changed_entries
if tuple(map(bool, names)) == pattern and funky.first_true(names)
]
return filtered_paths
@property
@funky.transform(set)
def removed_files(self):
return self._changed_entries_by_pattern(self.PATTERN_REMOVED) - self.ignored_files
@property
@funky.transform(set)
def added_files(self):
return self._changed_entries_by_pattern(self.PATTERN_ADDED) - self.ignored_files
@property
@funky.transform(set)
def modified_files(self):
modified_files = self._changed_entries_by_pattern(self.PATTERN_MODIFIED) - self.ignored_files
return modified_files
@property
@funky.transform(set)
def modified_unstaged_files(self):
timestamp = self.last_commit.commit_time
return [
f
for f in self.tracked_files
if os.stat(self.abspath(f)).st_mtime > timestamp
]
@property
def pending_files(self):
"""
Returns a list of all files that could be possibly staged
"""
# Union of both
return self.modified_files | self.added_files | self.removed_files
@property
def pending_files_by_state(self):
files = {
'modified': self.modified_files,
'added': self.added_files,
'removed': self.removed_files
}
# "Flip" the dictionary
return {
path: state
for state, paths in files.items()
for path in paths
}
"""
@property
@funky.transform(set)
def modified_files(self):
return self.modified_staged_files | self.modified_unstaged_files
"""
# Like: git add
@funky.arglist_method
def stage(self, files):
return self.repo.stage(files)
def add(self, *args, **kwargs):
return self.stage(*args, **kwargs)
# Like: git rm
@funky.arglist_method
def rm(self, files, force=False):
index = self.index
index_files = filter(lambda f: f in index, files)
for f in index_files:
del self.index[f]
return index.write()
def mv_fs(self, file_pair):
old_name, new_name = file_pair
os.rename(old_name, new_name)
# Like: git mv
@funky.arglist_method
def mv(self, files_pair):
index = self.index
files_in_index = filter(lambda f: f[0] in index, files_pair)
map(self.mv_fs, files_in_index)
old_files = map(funky.first, files_in_index)
new_files = map(funky.last, files_in_index)
self.add(new_files)
self.rm(old_files)
self.add(old_files)
return
@working_only
def _checkout_tree(self, tree):
return build_index_from_tree(
self.repo.path,
self.repo.index_path(),
self.repo.object_store,
tree
)
def checkout_all(self, commit_sha=None):
commit_sha = commit_sha or self.head
commit_tree = self._commit_tree(commit_sha)
# Rebuild index from the current tree
return self._checkout_tree(commit_tree)
def checkout(self, ref):
"""Checkout a given ref or SHA
"""
self.repo.refs.set_symbolic_ref('HEAD', ref)
commit_tree = self._commit_tree(ref)
# Rebuild index from the current tree
return self._checkout_tree(commit_tree)
@funky.arglist_method
def reset(self, files, commit='HEAD'):
pass
def rm_all(self):
# if we go at the index via the property, it is reconstructed
# each time and therefore clear() doesn't have the desired effect,
# therefore, we cache it in a variable and use that.
i = self.index
i.clear()
return i.write()
def _to_commit(self, commit_obj):
"""Allows methods to accept both SHA's or dulwich Commit objects as arguments
"""
if isinstance(commit_obj, basestring):
return self.repo[commit_obj]
return commit_obj
def _commit_sha(self, commit_obj):
"""Extracts a Dulwich commits SHA
"""
if utils.git.is_sha(commit_obj):
return commit_obj
elif isinstance(commit_obj, basestring):
# Can't use self[commit_obj] to avoid infinite recursion
commit_obj = self.repo[self.dwim_reference(commit_obj)]
return commit_obj.id
def dwim_reference(self, ref):
"""Dwim resolves a short reference to a full reference
"""
# Formats of refs we want to try in order
formats = [
"%s",
"refs/%s",
"refs/tags/%s",
"refs/heads/%s",
"refs/remotes/%s",
"refs/remotes/%s/HEAD",
]
for f in formats:
try:
fullref = f % ref
if not fullref in self.repo:
continue
return fullref
except:
continue
raise Exception("Could not resolve ref")
def blob_data(self, sha):
"""Return a blobs content for a given SHA
"""
return self[sha].data
# Get the nth parent back for a given commit
def get_parent_commit(self, commit, n=None):
""" Recursively gets the nth parent for a given commit
Warning: Remember that parents aren't the previous commits
"""
if n is None:
n = 1
commit = self._to_commit(commit)
parents = commit.parents
if n <= 0 or not parents:
# Return a SHA
return self._commit_sha(commit)
parent_sha = parents[0]
parent = self[parent_sha]
# Recur
return self.get_parent_commit(parent, n - 1)
def get_previous_commit(self, commit_ref, n=None):
commit_sha = self._parse_reference(commit_ref)
n = n or 1
commits = self.commits()
return funky.next(commits, commit_sha, n=n, default=commit_sha)
def _parse_reference(self, ref_string):
# COMMIT_REF~x
if '~' in ref_string:
ref, count = ref_string.split('~')
count = int(count)
commit_sha = self._commit_sha(ref)
return self.get_previous_commit(commit_sha, count)
return self._commit_sha(ref_string)
def _commit_tree(self, commit_sha):
"""Return the tree object for a given commit
"""
return self[commit_sha].tree
def diff(self, commit_sha, compare_to=None, diff_type=None, filter_binary=True):
diff_type = diff_type or self.DEFAULT_DIFF_TYPE
diff_func = self.DIFF_FUNCTIONS[diff_type]
if not compare_to:
compare_to = self.get_previous_commit(commit_sha)
return self._diff_between(compare_to, commit_sha, diff_function=diff_func)
def diff_working(self, ref=None, filter_binary=True):
"""Diff between the current working directory and the HEAD
"""
return utils.git.diff_changes_paths(
self.repo.object_store,
self.path,
self._changed_entries(ref=ref),
filter_binary=filter_binary
)
def get_commit_files(self, commit_sha, parent_path=None, is_tree=None, paths=None):
"""Returns a dict of the following Format :
{
"directory/filename.txt": {
'name': 'filename.txt',
'path': "directory/filename.txt",
"sha": "xxxxxxxxxxxxxxxxxxxx",
"data": "blablabla",
"mode": 0xxxxx",
},
...
}
"""
# Default values
context = {}
is_tree = is_tree or False
parent_path = parent_path or ''
if is_tree:
tree = self[commit_sha]
else:
tree = self[self._commit_tree(commit_sha)]
for entry in tree.items():
# Check if entry is a directory
if entry.mode == self.MODE_DIRECTORY:
context.update(
self.get_commit_files(entry.sha, parent_path=os.path.join(parent_path, entry.path), is_tree=True, paths=paths)
)
continue
subpath = os.path.join(parent_path, entry.path)
# Only add the files we want
if not(paths is None or subpath in paths):
continue
# Add file entry
context[subpath] = {
'name': entry.path,
'path': subpath,
'mode': entry.mode,
'sha': entry.sha,
'data': self.blob_data(entry.sha),
}
return context
def file_versions(self, path):
"""Returns all commits where given file was modified
"""
versions = []
commits_info = self.commit_info()
seen_shas = set()
for commit in commits_info:
try:
files = self.get_commit_files(commit['sha'], paths=[path])
file_path, file_data = files.items()[0]
except IndexError:
continue
file_sha = file_data['sha']
if file_sha in seen_shas:
continue
else:
seen_shas.add(file_sha)
# Add file info
commit['file'] = file_data
versions.append(file_data)
return versions
def _diff_between(self, old_commit_sha, new_commit_sha, diff_function=None, filter_binary=True):
"""Internal method for getting a diff between two commits
Please use .diff method unless you have very specific needs
"""
# If commit is first commit (new_commit_sha == old_commit_sha)
# then compare to an empty tree
if new_commit_sha == old_commit_sha:
old_tree = Tree()
else:
old_tree = self._commit_tree(old_commit_sha)
new_tree = self._commit_tree(new_commit_sha)
return diff_function(self.repo.object_store, old_tree, new_tree, filter_binary=filter_binary)
def changes(self, *args, **kwargs):
""" List of changes between two SHAs
Returns a list of lists of tuples :
[
[
(oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
],
...
]
"""
kwargs['diff_type'] = 'changes'
return self.diff(*args, **kwargs)
def changes_count(self, *args, **kwargs):
return len(self.changes(*args, **kwargs))
def _refs_by_pattern(self, pattern):
refs = self.refs
def item_filter(key_value):
"""Filter only concered refs"""
key, value = key_value
return key.startswith(pattern)
def item_map(key_value):
"""Rewrite keys"""
key, value = key_value
new_key = key[len(pattern):]
return (new_key, value)
return dict(
map(item_map,
filter(
item_filter,
refs.items()
)
)
)
@property
def refs(self):
return self.repo.get_refs()
def set_refs(refs_dict):
for k, v in refs_dict.items():
self.repo[k] = v
def import_refs(self, base, other):
return self.repo.refs.import_refs(base, other)
@property
def branches(self):
return self._refs_by_pattern(self.REFS_BRANCHES)
@property
def active_branch(self):
"""Returns the name of the active branch, or None, if HEAD is detached
"""
x = self.repo.refs.read_ref('HEAD')
if not x.startswith(SYMREF):
return None
else:
symref = x[len(SYMREF):]
if not symref.startswith(self.REFS_BRANCHES):
return None
else:
return symref[len(self.REFS_BRANCHES):]
@property
def active_sha(self):
"""Deprecated equivalent to head property
"""
return self.head
@property
def remote_branches(self):
return self._refs_by_pattern(self.REFS_REMOTES)
@property
def tags(self):
return self._refs_by_pattern(self.REFS_TAGS)
@property
def remotes(self):
""" Dict of remotes
{
'origin': 'http://friendco.de/some_user/repo.git',
...
}
"""
config = self.repo.get_config()
return {
keys[1]: values['url']
for keys, values in config.items()
if keys[0] == 'remote'
}
def add_remote(self, remote_name, remote_url):
# Get repo's config
config = self.repo.get_config()
# Add new entries for remote
config.set(('remote', remote_name), 'url', remote_url)
config.set(('remote', remote_name), 'fetch', "+refs/heads/*:refs/remotes/%s/*" % remote_name)
# Write to disk
config.write_to_path()
return remote_name
def add_ref(self, new_ref, old_ref):
self.repo.refs[new_ref] = old_ref
self.update_server_info()
def remove_ref(self, ref_name):
# Returns False if ref doesn't exist
if not ref_name in self.repo.refs:
return False
del self.repo.refs[ref_name]
self.update_server_info()
return True
def create_branch(self, base_branch, new_branch, tracking=None):
"""Try creating a new branch which tracks the given remote
if such a branch does not exist then branch off a local branch
"""
# The remote to track
tracking = self.DEFAULT_REMOTE
# Already exists
if new_branch in self.branches:
raise Exception("branch %s already exists" % new_branch)
# Get information about remote_branch
remote_branch = os.path.sep.join([tracking, base_branch])
# Fork Local
if base_branch in self.branches:
base_ref = self._format_ref_branch(base_branch)
# Fork remote
elif remote_branch in self.remote_branches:
base_ref = self._format_ref_remote(remote_branch)
# TODO : track
else:
raise Exception("Can not find the branch named '%s' to fork either locally or in '%s'" % (base_branch, tracking))
# Reference of new branch
new_ref = self._format_ref_branch(new_branch)
# Copy reference to create branch
self.add_ref(new_ref, base_ref)
return new_ref
def create_orphan_branch(self, new_branch, empty_index=None):
""" Create a new branch with no commits in it.
Technically, just points HEAD to a non-existent branch. The actual branch will
only be created if something is committed. This is equivalent to:
git checkout --orphan <new_branch>,
Unless empty_index is set to True, in which case the index will be emptied along
with the file-tree (which is always emptied). Against a clean working tree,
this is equivalent to:
git checkout --orphan <new_branch>
git reset --merge
"""
if new_branch in self.branches:
raise Exception("branch %s already exists" % new_branch)
new_ref = self._format_ref_branch(new_branch)
self.repo.refs.set_symbolic_ref('HEAD', new_ref)
if self.is_working:
if empty_index:
self.rm_all()
self.clean_working()
return new_ref
def remove_branch(self, branch_name):
ref = self._format_ref_branch(branch_name)
return self.remove_ref(ref)
def switch_branch(self, branch_name, tracking=None, create=None):
"""Changes the current branch
"""
if create is None:
create = True
# Check if branch exists
if not branch_name in self.branches:
self.create_branch(branch_name, branch_name, tracking=tracking)
# Get branch reference
branch_ref = self._format_ref_branch(branch_name)
# Change main branch
self.repo.refs.set_symbolic_ref('HEAD', branch_ref)
if self.is_working:
# Remove all files
self.clean_working()
# Add files for the current branch
self.checkout_all()
def create_tag(self, tag_name, target):
ref = self._format_ref_tag(tag_name)
return self.add_ref(ref, self._parse_reference(target))
def remove_tag(self, tag_name):
ref = self._format_ref_tag(tag_name)
return self.remove_ref(ref)
def clean(self, force=None, directories=None):
untracked_files = self.untracked_files
map(os.remove, untracked_files)
return untracked_files
def clean_working(self):
"""Purges all the working (removes everything except .git)
used by checkout_all to get clean branch switching
"""
return self.clean()
def _get_fs_structure(self, tree_sha, depth=None, parent_sha=None):
tree = self[tree_sha]
structure = {}
if depth is None:
depth = self.MAX_TREE_DEPTH
elif depth == 0:
return structure
for entry in tree.items():
# tree
if entry.mode == self.MODE_DIRECTORY:
# Recur
structure[entry.path] = self._get_fs_structure(entry.sha, depth=depth - 1, parent_sha=tree_sha)
# commit
else:
structure[entry.path] = entry.sha
structure['.'] = tree_sha
structure['..'] = parent_sha or tree_sha
return structure
def _get_fs_structure_by_path(self, tree_sha, path):
parts = path.split(os.path.sep)
depth = len(parts) + 1
structure = self._get_fs_structure(tree_sha, depth=depth)
return funky.subkey(structure, parts)
def commit_ls(self, ref, subpath=None):
"""List a "directory" for a given commit
using the tree of that commit
"""
tree_sha = self._commit_tree(ref)
# Root path
if subpath in self.ROOT_PATHS or not subpath:
return self._get_fs_structure(tree_sha, depth=1)
# Any other path
return self._get_fs_structure_by_path(tree_sha, subpath)
def commit_file(self, ref, path):
"""Return info on a given file for a given commit
"""
name, info = self.get_commit_files(ref, paths=[path]).items()[0]
return info
def commit_tree(self, ref, *args, **kwargs):
tree_sha = self._commit_tree(ref)
return self._get_fs_structure(tree_sha, *args, **kwargs)
def update_server_info(self):
if not self.is_bare:
return
update_server_info(self.repo)
def _is_fast_forward(self):
pass
def _merge_fast_forward(self):
pass
def __hash__(self):
"""This is required otherwise the memoize function will just mess it up
"""
return hash(self.path)
def __getitem__(self, key):
try:
sha = self._parse_reference(key)
except:
raise KeyError(key)
return self.repo[sha]
def __setitem__(self, key, value):
try:
key = self.dwim_reference(key)
except:
pass
self.repo[key] = value
def __contains__(self, key):
try:
key = self.dwim_reference(key)
except:
pass
return key in self.repo
def __delitem__(self, key):
try:
key = self.dwim_reference(key)
except:
raise KeyError(key)
self.remove_ref(key)
# Alias to clone_bare
fork = clone_bare
log = commit_info
diff_count = changes_count
contributors = recent_contributors
class Backend(BaseBackend):
default_branch = 'master'
null_revision = 'null'
file_mode = 0100644
directory_mode = 040755
@property
def repo(self):
if not hasattr(self, '_repo'):
self._repo = Repo(self.path)
return self._repo
def _get_commit(self, revision=None, branch=None):
repo = self.repo
if not revision:
try:
revision = repo.refs['refs/heads/%s' % branch]
except KeyError:
raise BranchDoesNotExist(self, branch)
elif isinstance(revision, DulwichCommit):
revision = revision.revision
try:
commit = repo[revision]
if not isinstance(commit, Commit):
raise CommitDoesNotExist(self, revision)
return commit
except KeyError:
raise CommitDoesNotExist(self, revision)
def _collect(self, tree, path, cache=None):
result = [(None, None, tree)]
bits = filter(None, path.split(os.path.sep))
repo = self.repo
for i, bit in enumerate(bits):
found = False
for mode, name, hexsha in tree.entries():
if name == bit:
found = True
if cache and hexsha in cache:
tree = cache[hexsha]
else:
tree = repo[hexsha]
result.append((mode, name, tree))
break
if not found:
result += [(self.directory_mode, bit, Tree())
for bit in bits[i:]]
break
return result
def _link(self, seq):
cache = {}
for i in xrange(len(seq) - 1, -1, -1):
mode, name, obj = seq[i]
cache[obj.id] = obj
if i > 0:
seq[i - 1][2][name] = (mode, obj.id)
return cache
### repo ###
def init_repo(self):
if os.path.exists(self.path):
return
os.mkdir(self.path)
self._repo = Repo.init_bare(self.path)
def delete_repo(self):
shutil.rmtree(self.path)
### branches ###
def has_branch(self, name):
return 'refs/heads/%s' % name in self.repo.refs
def create_branch(self, name, revision=None):
if self.has_branch(name):
raise BranchDoesAlreadyExist(self, name)
self.repo.refs['refs/heads/%s' % name] = self._get_commit(
revision, 'master').id
def delete_branch(self, name):
try:
del self.repo.refs['refs/heads/%s' % name]
except KeyError:
raise BranchDoesNotExist(self, name)
def rename_branch(self, old_name, new_name):
if old_name == new_name:
return
if not self.has_branch(old_name):
raise BranchDoesNotExist(self, old_name)
if self.has_branch(new_name):
raise BranchDoesAlreadyExist(new_name)
self.create_branch(new_name, 'refs/heads/%s' % old_name)
self.delete_branch(old_name)
### api ###
def revision(self, revision=None, branch='master'):
return DulwichCommit(self, self._get_commit(revision, branch))
def _walk(self, path, tree):
repo = self.repo
blobs, subtrees = [], []
for mode, name, hexsha in tree.entries():
if stat.S_ISREG(mode):
blobs.append(name)
elif stat.S_ISDIR(mode):
subtrees.append(name)
yield (path, subtrees, blobs)
for name in subtrees:
mode, hexsha = tree[name]
for t in self._walk(os.path.join(path, name), repo[hexsha]):
yield t
def walk(self, path, revision=None, branch='master'):
root = repo[self._get_commit(revision, branch).tree]
return self._walk(path, root)
def history(self,
path=None,
revision=None,
branch='master',
since_revision=None,
since=None,
sort=True):
if revision == self.null_revision:
return []
if path is not None:
path = clean_path(path)
if since_revision:
ctime = self.revision(since_revision).commit_time
if since:
since = max(ctime, since)
else:
since = ctime
if revision or branch:
pending = set([self._get_commit(revision, branch).id])
else:
pending = set(self._repo.get_refs().values())
visited = set()
result = []
repo = self.repo
while pending:
commit_id = pending.pop()
commit = self.revision(commit_id)
if commit_id in visited:
continue
visited.add(commit_id)
if since and since > commit.commit_time:
continue
if commit_id != since_revision:
pending.update(commit._commit.parents)
if path:
tree = repo[commit._commit.tree]
found = False
parents = commit._commit.parents
for parent in parents:
parent_tree = repo[repo[parent].tree]
if not is_same_object(repo, tree, parent_tree, path):
found = True
break
if not parents and get_by_path(repo, tree, path):
found = True
if not found:
continue
result.append(commit)
if sort:
result.sort(key=attrgetter('commit_time'), reverse=True)
return result
def do_read(self, path, revision=None, branch='master'):
path = clean_path(path)
repo = self.repo
if revision == self.null_revision:
raise FileDoesNotExist(
self, "'%s' does not exist at revision null" % path)
c = self._get_commit(revision, branch)
obj = get_by_path(repo, repo[c.tree], path)
if not obj:
raise FileDoesNotExist(self, "'%s' does not exist" % path)
if not isinstance(obj, Blob):
raise FileDoesNotExist(self, "'%s' is not a regular file" % path)
data = obj.as_pretty_string()
return data
def do_commit(self,
message='',
author=None,
committer=None,
branch='master',
parent=None):
if isinstance(message, unicode):
message = message.encode('utf-8')
repo = self.repo
try:
parent = self._get_commit(parent, branch)
root = repo[parent.tree]
except BranchDoesNotExist:
if branch == 'master': # initial commit
root = Tree()
else:
raise
cache = {}
paths = set()
objects = set()
for path, (action, data) in self.changes.iteritems():
path = clean_path(path)
paths.add(path)
dirname, filename = os.path.split(path)
trees = self._collect(root, dirname, cache)
if action == WRITE:
blob = Blob.from_string(data)
trees[-1][2][filename] = (self.file_mode, blob.id)
cache[blob.id] = blob
elif action == DELETE:
del trees[-1][2][filename]
elif action == RENAME:
old = self._collect(root, data, cache)
mode, name, obj = old[-1]
del old[-2][2][name]
trees[-1][2][filename] = (mode, obj.id)
cache.update(self._link(old[:-1]))
paths.add(data)
cache.update(self._link(trees))
else:
objects.add(root)
# collect all objects that have to be committed
for path in paths:
objects.update(
[obj for mode, name, obj in self._collect(root, path, cache)])
# create the commit
c = Commit()
if parent:
c.parents = [parent.id]
c.tree = root.id
c.committer = committer or self.committer
c.author = author or c.committer
t = time.localtime()
c.commit_time = c.author_time = int(time.mktime(t))
c.commit_timezone = c.author_timezone = t.tm_isdst * 3600 - time.timezone
c.encoding = "UTF-8"
c.message = message
objects.add(c)
# write everything to disk
for obj in objects:
repo.object_store.add_object(obj)
repo.refs['refs/heads/%s' % branch] = c.id
return DulwichCommit(self, c)
#!/usr/bin/env python2
import os.path
import sys
import glob
from email.utils import formatdate
from dulwich.repo import Repo
from dulwich.objects import Blob, Tree, Commit
repo = Repo(".")
import sys
if len(sys.argv) > 1:
commit_sha = repo.get_refs()["refs/heads/"+sys.argv[1]]
else:
commit_sha = repo.head()
commit = repo.get_object(commit_sha)
index = repo.open_index()
assert not list(index.changes_from_tree(repo.object_store, commit.tree)), "uncommited changes"
def add_blob(store, path):
with open(path, "rb") as f:
blob = Blob.from_string(f.read())
store.add_object(blob)
return blob.id
def add_files(store, tree, files):
class Backend(BaseBackend):
default_branch = 'master'
null_revision = 'null'
file_mode = 0100644
directory_mode = 040755
@property
def repo(self):
if not hasattr(self, '_repo'):
self._repo = Repo(self.path)
return self._repo
def _get_commit(self, revision=None, branch=None):
repo = self.repo
if not revision:
try:
revision = repo.refs['refs/heads/%s' % branch]
except KeyError:
raise BranchDoesNotExist(self, branch)
elif isinstance(revision, DulwichCommit):
revision = revision.revision
try:
commit = repo[revision]
if not isinstance(commit, Commit):
raise CommitDoesNotExist(self, revision)
return commit
except KeyError:
raise CommitDoesNotExist(self, revision)
def _collect(self, tree, path, cache=None):
result = [(None, None, tree)]
bits = filter(None, path.split(os.path.sep))
repo = self.repo
for i, bit in enumerate(bits):
found = False
for mode, name, hexsha in tree.entries():
if name == bit:
found = True
if cache and hexsha in cache:
tree = cache[hexsha]
else:
tree = repo[hexsha]
result.append((mode, name, tree))
break
if not found:
result += [(self.directory_mode, bit, Tree()) for bit in bits[i:]]
break
return result
def _link(self, seq):
cache = {}
for i in xrange(len(seq) - 1, -1, -1):
mode, name, obj = seq[i]
cache[obj.id] = obj
if i > 0:
seq[i - 1][2][name] = (mode, obj.id)
return cache
### repo ###
def init_repo(self):
if os.path.exists(self.path):
return
os.mkdir(self.path)
self._repo = Repo.init_bare(self.path)
def delete_repo(self):
shutil.rmtree(self.path)
### branches ###
def has_branch(self, name):
return 'refs/heads/%s' % name in self.repo.refs
def create_branch(self, name, revision=None):
if self.has_branch(name):
raise BranchDoesAlreadyExist(self, name)
self.repo.refs['refs/heads/%s' % name] = self._get_commit(revision, 'master').id
def delete_branch(self, name):
try:
del self.repo.refs['refs/heads/%s' % name]
except KeyError:
raise BranchDoesNotExist(self, name)
def rename_branch(self, old_name, new_name):
if old_name == new_name:
return
if not self.has_branch(old_name):
raise BranchDoesNotExist(self, old_name)
if self.has_branch(new_name):
raise BranchDoesAlreadyExist(new_name)
self.create_branch(new_name, 'refs/heads/%s' % old_name)
self.delete_branch(old_name)
### api ###
def revision(self, revision=None, branch='master'):
return DulwichCommit(self, self._get_commit(revision, branch))
def _walk(self, path, tree):
repo = self.repo
blobs, subtrees = [], []
for mode, name, hexsha in tree.entries():
if stat.S_ISREG(mode):
blobs.append(name)
elif stat.S_ISDIR(mode):
subtrees.append(name)
yield (path, subtrees, blobs)
for name in subtrees:
mode, hexsha = tree[name]
for t in self._walk(os.path.join(path, name), repo[hexsha]):
yield t
def walk(self, path, revision=None, branch='master'):
root = repo[self._get_commit(revision, branch).tree]
return self._walk(path, root)
def history(self, path=None, revision=None, branch='master', since_revision=None, since=None, sort=True):
if revision == self.null_revision:
return []
if path is not None:
path = clean_path(path)
if since_revision:
ctime = self.revision(since_revision).commit_time
if since:
since = max(ctime, since)
else:
since = ctime
if revision or branch:
pending = set([self._get_commit(revision, branch).id])
else:
pending = set(self._repo.get_refs().values())
visited = set()
result = []
repo = self.repo
while pending:
commit_id = pending.pop()
commit = self.revision(commit_id)
if commit_id in visited:
continue
visited.add(commit_id)
if since and since > commit.commit_time:
continue
if commit_id != since_revision:
pending.update(commit._commit.parents)
if path:
tree = repo[commit._commit.tree]
found = False
parents = commit._commit.parents
for parent in parents:
parent_tree = repo[repo[parent].tree]
if not is_same_object(repo, tree, parent_tree, path):
found = True
break
if not parents and get_by_path(repo, tree, path):
found = True
if not found:
continue
result.append(commit)
if sort:
result.sort(key=attrgetter('commit_time'), reverse=True)
return result
def do_read(self, path, revision=None, branch='master'):
path = clean_path(path)
repo = self.repo
if revision == self.null_revision:
raise FileDoesNotExist(self, "'%s' does not exist at revision null" % path)
c = self._get_commit(revision, branch)
obj = get_by_path(repo, repo[c.tree], path)
if not obj:
raise FileDoesNotExist(self, "'%s' does not exist" % path)
if not isinstance(obj, Blob):
raise FileDoesNotExist(self, "'%s' is not a regular file" % path)
data = obj.as_pretty_string()
return data
def do_commit(self, message='', author=None, committer=None, branch='master', parent=None):
if isinstance(message, unicode):
message = message.encode('utf-8')
repo = self.repo
try:
parent = self._get_commit(parent, branch)
root = repo[parent.tree]
except BranchDoesNotExist:
if branch == 'master': # initial commit
root = Tree()
else:
raise
cache = {}
paths = set()
objects = set()
for path, (action, data) in self.changes.iteritems():
path = clean_path(path)
paths.add(path)
dirname, filename = os.path.split(path)
trees = self._collect(root, dirname, cache)
if action == WRITE:
blob = Blob.from_string(data)
trees[-1][2][filename] = (self.file_mode, blob.id)
cache[blob.id] = blob
elif action == DELETE:
del trees[-1][2][filename]
elif action == RENAME:
old = self._collect(root, data, cache)
mode, name, obj = old[-1]
del old[-2][2][name]
trees[-1][2][filename] = (mode, obj.id)
cache.update(self._link(old[:-1]))
paths.add(data)
cache.update(self._link(trees))
else:
objects.add(root)
# collect all objects that have to be committed
for path in paths:
objects.update([obj for mode, name, obj in self._collect(root, path, cache)])
# create the commit
c = Commit()
if parent:
c.parents = [parent.id]
c.tree = root.id
c.committer = committer or self.committer
c.author = author or c.committer
t = time.localtime()
c.commit_time = c.author_time = int(time.mktime(t))
c.commit_timezone = c.author_timezone = t.tm_isdst * 3600 - time.timezone
c.encoding = "UTF-8"
c.message = message
objects.add(c)
# write everything to disk
for obj in objects:
repo.object_store.add_object(obj)
repo.refs['refs/heads/%s' % branch] = c.id
return DulwichCommit(self, c)
class Gits3(object):
def __init__(self, path):
self.path = path
self.open_repo(path)
def open_repo(self, path):
self.repo = Repo(path)
def get_id(self, ref):
return self.repo.get_refs()[ref]
def get_updates(self, local_ref, tracking_ref):
refs = self.repo.get_refs()
for key, value in refs.iteritems():
print key, value
local = refs[local_ref]
try:
remote = refs[tracking_ref]
except KeyError:
remote = None
if local == remote:
return None
local_object = self.repo.get_object(local)
commits = self.get_commits(local_object, [remote])
objects = self.get_objects(commits)
print objects
if remote:
remote_object = self.repo.get_object(remote)
filtered_objects = self.filter_objects(objects, remote_object)
else:
filtered_objects = objects
filtered_objects = set(filtered_objects)
return filtered_objects
def filter_objects(self, objects, old_commit):
filtered = []
old_treeId = old_commit.tree
old_objects = self.get_objects_in_tree(old_treeId)
for object in objects:
if object not in old_objects:
filtered.append(object)
return filtered
def get_commits(self, interesting, uninteresting):
commits = [interesting]
remaining = interesting.get_parents()
while remaining:
pId = remaining.pop(0)
if pId in uninteresting:
continue
else:
parent = self.repo.get_object(pId)
commits.append(parent)
parents = parent.get_parents()
remaining.extend(parents)
return commits
def get_objects(self, commits):
objects = []
while commits:
commit = commits.pop(0)
objects.append(commit)
objects.extend(self.get_objects_in_tree(commit.tree))
return objects
def get_objects_in_tree(self, treeId):
objects = []
tree = self.repo.get_object(treeId)
objects.append(tree)
entries = tree.entries()
for entryId in entries:
# get the entry's sha
objectId = entryId[2]
object = self.repo.get_object(objectId)
if isinstance(object, Tree):
objects.extend(self.get_objects_in_tree(objectId))
else:
objects.append(object)
return objects
def generate_pack_name(self, objects):
m = hashlib.sha1()
for object in objects:
sha1 = object.sha().hexdigest()
# print sha1
m.update(sha1)
file_name = m.hexdigest()
# print 'File Name is ', file_name
return file_name
def write_pack(self, pack_name, objects):
write_pack('pack-' + pack_name, [(x, "") for x in objects],
len(objects))
def find_tracking_ref_names(self, fetch, refs):
if fetch[0] == '+':
fetch = fetch[1:]
tmp = fetch.split(':')
src = tmp[0]
dst = tmp[1]
# TODO double check that both src and dst have wild cards, or both don't
# match the source with refs
if src.endswith('*') and refs.startswith(src[:-1]):
return self.expand_from_src(src, dst, refs)
else:
return dst
def expand_from_src(self, src, dst, refs):
return dst[:-1] + refs[len(src)-1:]
class GitHandler(object):
def __init__(self, dest_repo, ui):
self.repo = dest_repo
self.ui = ui
self.mapfile = 'git-mapfile'
self.configfile = 'git-config'
if ui.config('git', 'intree'):
self.gitdir = self.repo.wjoin('.git')
else:
self.gitdir = self.repo.join('git')
self.importbranch = ui.config('git', 'importbranch')
self.exportbranch = ui.config('git', 'exportbranch', 'refs/heads/master')
self.bookbranch = ui.config('git', 'bookbranch', '')
self.init_if_missing()
self.load_git()
self.load_map()
self.load_config()
# make the git data directory
def init_if_missing(self):
if not os.path.exists(self.gitdir):
os.mkdir(self.gitdir)
Repo.init_bare(self.gitdir)
def load_git(self):
self.git = Repo(self.gitdir)
## FILE LOAD AND SAVE METHODS
def map_set(self, gitsha, hgsha):
self._map_git[gitsha] = hgsha
self._map_hg[hgsha] = gitsha
def map_hg_get(self, gitsha):
return self._map_git.get(gitsha)
def map_git_get(self, hgsha):
return self._map_hg.get(hgsha)
def load_map(self):
self._map_git = {}
self._map_hg = {}
if os.path.exists(self.repo.join(self.mapfile)):
for line in self.repo.opener(self.mapfile):
gitsha, hgsha = line.strip().split(' ', 1)
self._map_git[gitsha] = hgsha
self._map_hg[hgsha] = gitsha
def save_map(self):
file = self.repo.opener(self.mapfile, 'w+', atomictemp=True)
for gitsha, hgsha in sorted(self._map_git.iteritems()):
file.write("%s %s\n" % (gitsha, hgsha))
file.rename()
def load_config(self):
self._config = {}
if os.path.exists(self.repo.join(self.configfile)):
for line in self.repo.opener(self.configfile):
key, value = line.strip().split(' ', 1)
self._config[key] = value
def save_config(self):
file = self.repo.opener(self.configfile, 'w+', atomictemp=True)
for key, value in self._config.iteritems():
file.write("%s %s\n" % (key, value))
file.rename()
## END FILE LOAD AND SAVE METHODS
def import_commits(self, remote_name):
self.import_git_objects(remote_name)
self.save_map()
def fetch(self, remote_name):
self.ui.status(_("fetching from : %s\n") % remote_name)
self.export_git_objects()
refs = self.fetch_pack(remote_name)
if refs:
self.import_git_objects(remote_name, refs)
self.import_local_tags(refs)
self.save_map()
def export_commits(self):
self.export_git_objects()
self.export_hg_tags()
self.update_references()
self.save_map()
def push(self, remote_name):
self.ui.status(_("pushing to : %s\n") % remote_name)
self.export_commits()
self.update_remote_references(remote_name)
self.upload_pack(remote_name)
def remote_add(self, remote_name, git_url):
self._config['remote.' + remote_name + '.url'] = git_url
self.save_config()
def remote_remove(self, remote_name):
key = 'remote.' + remote_name + '.url'
if key in self._config:
del self._config[key]
self.save_config()
def remote_show(self, remote_name):
key = 'remote.' + remote_name + '.url'
if key in self._config:
name = self._config[key]
self.ui.status(_("URL for %s : %s\n") % (remote_name, name, ))
else:
self.ui.status(_("No remote named : %s\n") % remote_name)
return
def remote_list(self):
for key, value in self._config.iteritems():
if key[0:6] == 'remote':
self.ui.status('%s\t%s\n' % (key, value, ))
def remote_name_to_url(self, remote_name):
return self._config['remote.' + remote_name + '.url']
def update_references(self):
try:
# We only care about bookmarks of the form 'name',
# not 'remote/name'.
def is_local_ref(item): return item[0].count('/') == 0
bms = bookmarks.parse(self.repo)
bms = dict(filter(is_local_ref, bms.items()))
# Create a local Git branch name for each
# Mercurial bookmark.
for key in bms:
hg_sha = hex(bms[key])
git_sha = self.map_git_get(hg_sha)
self.git.set_ref('refs/heads/' + key, git_sha)
except AttributeError:
# No bookmarks extension
pass
c = self.map_git_get(hex(self.repo.changelog.tip()))
self.git.set_ref(self.exportbranch, c)
def export_hg_tags(self):
for tag, sha in self.repo.tags().iteritems():
if tag[-3:] == '^{}':
continue
if tag == 'tip':
continue
self.git.set_ref('refs/tags/' + tag, self.map_git_get(hex(sha)))
# Make sure there's a refs/remotes/remote_name/name
# for every refs/heads/name
def update_remote_references(self, remote_name):
self.git.set_remote_refs(self.local_heads(), remote_name)
def local_heads(self):
def is_local_head(item): return item[0].startswith('refs/heads')
refs = self.git.get_refs()
return dict(filter(is_local_head, refs.items()))
def export_git_objects(self):
self.ui.status(_("importing Hg objects into Git\n"))
total = len(self.repo.changelog)
if total:
magnitude = int(math.log(total, 10)) + 1
else:
magnitude = 1
for i, rev in enumerate(self.repo.changelog):
if i%100 == 0:
self.ui.status(_("at: %*d/%d\n") % (magnitude, i, total))
ctx = self.repo.changectx(rev)
state = ctx.extra().get('hg-git', None)
if state == 'octopus':
self.ui.debug("revision %d is a part of octopus explosion\n" % rev)
continue
pgit_sha, already_written = self.export_hg_commit(rev)
if not already_written:
self.save_map()
# convert this commit into git objects
# go through the manifest, convert all blobs/trees we don't have
# write the commit object (with metadata info)
def export_hg_commit(self, rev):
def is_octopus_part(ctx):
return ctx.extra().get('hg-git', None) in set(['octopus', 'octopus-done'])
# return if we've already processed this
node = self.repo.changelog.lookup(rev)
phgsha = hex(node)
pgit_sha = self.map_git_get(phgsha)
if pgit_sha:
return pgit_sha, True
self.ui.status(_("converting revision %s\n") % str(rev))
# make sure parents are converted first
ctx = self.repo.changectx(rev)
extra = ctx.extra()
parents = []
if extra.get('hg-git', None) == 'octopus-done':
# implode octopus parents
part = ctx
while is_octopus_part(part):
(p1, p2) = part.parents()
assert not is_octopus_part(p1)
parents.append(p1)
part = p2
parents.append(p2)
else:
parents = ctx.parents()
for parent in parents:
p_rev = parent.rev()
hgsha = hex(parent.node())
git_sha = self.map_git_get(hgsha)
if not p_rev == -1:
if not git_sha:
self.export_hg_commit(p_rev)
tree_sha, renames = self.write_git_tree(ctx)
commit = {}
commit['tree'] = tree_sha
(time, timezone) = ctx.date()
# hg authors might not have emails
author = ctx.user()
if not '>' in author:
author = author + ' <none@none>'
commit['author'] = author + ' ' + str(int(time)) + ' ' + format_timezone(-timezone)
message = ctx.description()
commit['message'] = ctx.description() + "\n"
if 'committer' in extra:
# fixup timezone
(name_timestamp, timezone) = extra['committer'].rsplit(' ', 1)
try:
timezone = format_timezone(-int(timezone))
commit['committer'] = '%s %s' % (name_timestamp, timezone)
except ValueError:
self.ui.warn(_("Ignoring committer in extra, invalid timezone in r%s: '%s'.\n") % (rev, timezone))
if 'encoding' in extra:
commit['encoding'] = extra['encoding']
# HG EXTRA INFORMATION
add_extras = False
extra_message = ''
if not ctx.branch() == 'default':
add_extras = True
extra_message += "branch : " + ctx.branch() + "\n"
if renames:
add_extras = True
for oldfile, newfile in renames:
extra_message += "rename : " + oldfile + " => " + newfile + "\n"
for key, value in extra.iteritems():
if key in ['committer', 'encoding', 'branch', 'hg-git', 'git']:
continue
else:
add_extras = True
extra_message += "extra : " + key + " : " + urllib.quote(value) + "\n"
if add_extras:
commit['message'] += "\n--HG--\n" + extra_message
commit['parents'] = []
for parent in parents:
hgsha = hex(parent.node())
git_sha = self.map_git_get(hgsha)
if git_sha:
commit['parents'].append(git_sha)
commit_sha = self.git.write_commit_hash(commit) # writing new blobs to git
self.map_set(commit_sha, phgsha)
return commit_sha, False
def write_git_tree(self, ctx):
trees = {}
man = ctx.manifest()
renames = []
for filenm in man.keys():
# write blob if not in our git database
fctx = ctx.filectx(filenm)
rename = fctx.renamed()
if rename:
filerename, sha = rename
renames.append((filerename, filenm))
is_exec = 'x' in fctx.flags()
is_link = 'l' in fctx.flags()
file_id = hex(fctx.filenode())
blob_sha = self.map_git_get(file_id)
if not blob_sha:
blob_sha = self.git.write_blob(fctx.data()) # writing new blobs to git
self.map_set(blob_sha, file_id)
parts = filenm.split('/')
if len(parts) > 1:
# get filename and path for leading subdir
filepath = parts[-1:][0]
dirpath = "/".join([v for v in parts[0:-1]]) + '/'
# get subdir name and path for parent dir
parpath = '/'
nparpath = '/'
for part in parts[0:-1]:
if nparpath == '/':
nparpath = part + '/'
else:
nparpath += part + '/'
treeentry = ['tree', part + '/', nparpath]
if parpath not in trees:
trees[parpath] = []
if treeentry not in trees[parpath]:
trees[parpath].append( treeentry )
parpath = nparpath
# set file entry
fileentry = ['blob', filepath, blob_sha, is_exec, is_link]
if dirpath not in trees:
trees[dirpath] = []
trees[dirpath].append(fileentry)
else:
fileentry = ['blob', parts[0], blob_sha, is_exec, is_link]
if '/' not in trees:
trees['/'] = []
trees['/'].append(fileentry)
dirs = trees.keys()
if dirs:
# sort by tree depth, so we write the deepest trees first
dirs.sort(lambda a, b: len(b.split('/'))-len(a.split('/')))
dirs.remove('/')
dirs.append('/')
else:
# manifest is empty => make empty root tree
trees['/'] = []
dirs = ['/']
# write all the trees
tree_sha = None
tree_shas = {}
for dirnm in dirs:
tree_data = []
for entry in trees[dirnm]:
# replace tree path with tree SHA
if entry[0] == 'tree':
sha = tree_shas[entry[2]]
entry[2] = sha
tree_data.append(entry)
tree_sha = self.git.write_tree_array(tree_data) # writing new trees to git
tree_shas[dirnm] = tree_sha
return (tree_sha, renames) # should be the last root tree sha
def remote_head(self, remote_name):
for head, sha in self.git.remote_refs(remote_name).iteritems():
if head == 'HEAD':
return self.map_hg_get(sha)
return None
def upload_pack(self, remote_name):
git_url = self.remote_name_to_url(remote_name)
client, path = self.get_transport_and_path(git_url)
changed = self.get_changed_refs
genpack = self.generate_pack_contents
try:
self.ui.status(_("creating and sending data\n"))
changed_refs = client.send_pack(path, changed, genpack)
if changed_refs:
new_refs = {}
for ref, sha in changed_refs.iteritems():
self.ui.status(" "+ remote_name + "::" + ref + " => GIT:" + sha[0:8] + "\n")
new_refs[ref] = sha
self.git.set_remote_refs(new_refs, remote_name)
self.update_hg_bookmarks(remote_name)
except:
# TODO : remove try/except or do something useful here
raise
# TODO : for now, we'll just push all heads that match remote heads
# * we should have specified push, tracking branches and --all
# takes a dict of refs:shas from the server and returns what should be
# pushed up
def get_changed_refs(self, refs):
keys = refs.keys()
changed = {}
if not keys:
return None
# TODO : this is a huge hack
if keys[0] == 'capabilities^{}': # nothing on the server yet - first push
changed['refs/heads/master'] = self.git.ref('master')
tags = self.git.get_tags()
for tag, sha in tags.iteritems():
tag_name = 'refs/tags/' + tag
if tag_name not in refs:
changed[tag_name] = sha
for ref_name in keys:
parts = ref_name.split('/')
if parts[0] == 'refs': # strip off 'refs/heads'
if parts[1] == 'heads':
head = "/".join([v for v in parts[2:]])
local_ref = self.git.ref(ref_name)
if local_ref:
if not local_ref == refs[ref_name]:
changed[ref_name] = local_ref
# Also push any local branches not on the server yet
for head in self.local_heads():
if not head in refs:
ref = self.git.ref(head)
changed[head] = ref
return changed
# takes a list of shas the server wants and shas the server has
# and generates a list of commit shas we need to push up
def generate_pack_contents(self, want, have):
graph_walker = SimpleFetchGraphWalker(want, self.git.get_parents)
next = graph_walker.next()
shas = set()
while next:
if next in have:
graph_walker.ack(next)
else:
shas.add(next)
next = graph_walker.next()
seen = []
# so now i have the shas, need to turn them into a list of
# tuples (sha, path) for ALL the objects i'm sending
# TODO : don't send blobs or trees they already have
def get_objects(tree, path):
changes = list()
changes.append((tree, path))
for (mode, name, sha) in tree.entries():
if mode == 0160000: # TODO : properly handle submodules and document what 57344 means
continue
if sha in seen:
continue
obj = self.git.get_object(sha)
seen.append(sha)
if isinstance (obj, Blob):
changes.append((obj, path + name))
elif isinstance(obj, Tree):
changes.extend(get_objects(obj, path + name + '/'))
return changes
objects = []
for commit_sha in shas:
commit = self.git.commit(commit_sha)
objects.append((commit, 'commit'))
tree = self.git.get_object(commit.tree)
objects.extend( get_objects(tree, '/') )
return objects
def fetch_pack(self, remote_name):
git_url = self.remote_name_to_url(remote_name)
client, path = self.get_transport_and_path(git_url)
graphwalker = SimpleFetchGraphWalker(self.git.heads().values(), self.git.get_parents)
f, commit = self.git.object_store.add_pack()
try:
determine_wants = self.git.object_store.determine_wants_all
refs = client.fetch_pack(path, determine_wants, graphwalker, f.write, sys.stdout.write)
f.close()
commit()
if refs:
self.git.set_remote_refs(refs, remote_name)
else:
self.ui.status(_("nothing new on the server\n"))
return refs
except:
f.close()
raise
# take refs just fetched, add local tags for all tags not in .hgtags
def import_local_tags(self, refs):
keys = refs.keys()
if not keys:
return None
for k in keys[0:]:
ref_name = k
parts = k.split('/')
if (parts[0] == 'refs' and parts[1] == 'tags'):
ref_name = "/".join([v for v in parts[2:]])
if ref_name[-3:] == '^{}':
ref_name = ref_name[:-3]
if not ref_name in self.repo.tags():
obj = self.git.get_object(refs[k])
sha = None
if isinstance (obj, Commit): # lightweight
sha = self.map_hg_get(refs[k])
if isinstance (obj, Tag): # annotated
(obj_type, obj_sha) = obj.get_object()
obj = self.git.get_object(obj_sha)
if isinstance (obj, Commit):
sha = self.map_hg_get(obj_sha)
if sha:
self.repo.tag(ref_name, hex_to_sha(sha), '', True, None, None)
def import_git_objects(self, remote_name=None, refs=None):
self.ui.status(_("importing Git objects into Hg\n"))
# import heads and fetched tags as remote references
todo = []
done = set()
convert_list = {}
self.renames = {}
# get a list of all the head shas
if refs:
for head, sha in refs.iteritems():
todo.append(sha)
else:
if remote_name:
todo = self.git.remote_refs(remote_name).values()[:]
elif self.importbranch:
branches = self.importbranch.split(',')
todo = [self.git.ref(i.strip()) for i in branches]
else:
todo = self.git.heads().values()[:]
# traverse the heads getting a list of all the unique commits
while todo:
sha = todo.pop()
assert isinstance(sha, str)
if sha in done:
continue
done.add(sha)
obj = self.git.get_object(sha)
if isinstance (obj, Commit):
convert_list[sha] = obj
todo.extend([p for p in obj.parents if p not in done])
if isinstance(obj, Tag):
(obj_type, obj_sha) = obj.get_object()
obj = self.git.get_object(obj_sha)
if isinstance (obj, Commit):
convert_list[sha] = obj
todo.extend([p for p in obj.parents if p not in done])
# sort the commits
commits = toposort.TopoSort(convert_list).items()
# import each of the commits, oldest first
total = len(commits)
magnitude = int(math.log(total, 10)) + 1 if total else 1
for i, csha in enumerate(commits):
if i%100 == 0:
self.ui.status(_("at: %*d/%d\n") % (magnitude, i, total))
commit = convert_list[csha]
if not self.map_hg_get(csha): # it's already here
self.import_git_commit(commit)
else:
# we need to get rename info for further upstream
self.pseudo_import_git_commit(commit)
self.update_hg_bookmarks(remote_name)
def update_hg_bookmarks(self, remote_name):
try:
bms = bookmarks.parse(self.repo)
if remote_name:
heads = self.git.remote_refs(remote_name)
else:
branches = self.bookbranch.split(',')
heads = dict((i, self.git.ref(i.strip())) for i in branches)
base_name = (remote_name + '/') if remote_name else ''
for head, sha in heads.iteritems():
if not sha:
self.ui.warn(_("Could not resolve head %s.\n") % head)
continue
hgsha = hex_to_sha(self.map_hg_get(sha))
if not head == 'HEAD':
bms[base_name + head] = hgsha
if heads:
bookmarks.write(self.repo, bms)
except AttributeError:
self.ui.warn(_('creating bookmarks failed, do you have'
' bookmarks enabled?\n'))
def convert_git_int_mode(self, mode):
# TODO : make these into constants
convert = {
0100644: '',
0100755: 'x',
0120000: 'l'}
if mode in convert:
return convert[mode]
return ''
def extract_hg_metadata(self, message):
split = message.split("\n\n--HG--\n", 1)
renames = {}
extra = {}
files = []
branch = False
if len(split) == 2:
message, meta = split
lines = meta.split("\n")
for line in lines:
if line == '':
continue
command, data = line.split(" : ", 1)
if command == 'rename':
before, after = data.split(" => ", 1)
renames[after] = before
if command == 'branch':
branch = data
if command == 'files':
files.append(data)
if command == 'extra':
before, after = data.split(" : ", 1)
extra[before] = urllib.unquote(after)
return (message, renames, branch, files, extra)
def pseudo_import_git_commit(self, commit):
(strip_message, hg_renames, hg_branch) = self.extract_hg_metadata(commit.message)
cs = self.map_hg_get(commit.id)
p1 = nullid
p2 = nullid
if len(commit.parents) > 0:
sha = commit.parents[0]
p1 = self.map_hg_get(sha)
if len(commit.parents) > 1:
sha = commit.parents[1]
p2 = self.map_hg_get(sha)
if len(commit.parents) > 2:
# TODO : map extra parents to the extras file
pass
# saving rename info
if (not (p2 == nullid) or (p1 == nullid)):
self.renames[cs] = {}
else:
self.renames[cs] = self.renames[p1].copy()
self.renames[cs].update(hg_renames)
def import_git_commit(self, commit):
self.ui.debug(_("importing: %s\n") % commit.id)
# TODO : Do something less coarse-grained than try/except on the
# get_file call for removed files
(strip_message, hg_renames, hg_branch, files, extra) = self.extract_hg_metadata(commit.message)
# get a list of the changed, added, removed files
files = self.git.get_files_changed(commit)
date = (commit.author_time, -commit.author_timezone)
text = strip_message
def getfilectx(repo, memctx, f):
try:
(mode, sha, data) = self.git.get_file(commit, f)
e = self.convert_git_int_mode(mode)
except TypeError:
raise IOError()
if f in hg_renames:
copied_path = hg_renames[f]
else:
copied_path = None
return context.memfilectx(f, data, 'l' in e, 'x' in e, copied_path)
gparents = map(self.map_hg_get, commit.parents)
p1, p2 = (nullid, nullid)
octopus = False
if len(gparents) > 1:
# merge, possibly octopus
def commit_octopus(p1, p2):
ctx = context.memctx(self.repo, (p1, p2), text, files, getfilectx,
commit.author, date, {'hg-git': 'octopus'})
return hex(self.repo.commitctx(ctx))
octopus = len(gparents) > 2
p2 = gparents.pop()
p1 = gparents.pop()
while len(gparents) > 0:
p2 = commit_octopus(p1, p2)
p1 = gparents.pop()
else:
if gparents:
p1 = gparents.pop()
# wierd hack for explicit file renames in first but not second branch
if not (p2 == nullid):
vals = [item for item in self.renames[p1].values() if not item in self.renames[p2].values()]
for removefile in vals:
files.remove(removefile)
author = commit.author
extra = {}
if ' <none@none>' in commit.author:
author = commit.author[:-12]
# if named branch, add to extra
if hg_branch:
extra['branch'] = hg_branch
# if committer is different than author, add it to extra
if not commit._author_raw == commit._committer_raw:
extra['committer'] = "%s %d %d" % (commit.committer, commit.commit_time, -commit.commit_timezone)
if commit._encoding:
extra['encoding'] = commit._encoding
if hg_branch:
extra['branch'] = hg_branch
if octopus:
extra['hg-git'] ='octopus-done'
ctx = context.memctx(self.repo, (p1, p2), text, files, getfilectx,
author, date, extra)
node = self.repo.commitctx(ctx)
# save changeset to mapping file
cs = hex(node)
self.map_set(commit.id, cs)
# saving rename info
if (not (p2 == nullid) or (p1 == nullid)):
self.renames[cs] = {}
else:
self.renames[cs] = self.renames[p1].copy()
self.renames[cs].update(hg_renames)
def check_bookmarks(self):
if self.ui.config('extensions', 'hgext.bookmarks') is not None:
self.ui.warn("YOU NEED TO SETUP BOOKMARKS\n")
def get_transport_and_path(self, uri):
from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
for handler, transport in (("git://", TCPGitClient), ("git@", SSHGitClient), ("git+ssh://", SSHGitClient)):
if uri.startswith(handler):
if handler == 'git@':
host, path = uri[len(handler):].split(":", 1)
host = 'git@' + host
else:
host, path = uri[len(handler):].split("/", 1)
return transport(host), '/' + path
# if its not git or git+ssh, try a local url..
return SubprocessGitClient(), uri
def clear(self):
mapfile = self.repo.join(self.mapfile)
if os.path.exists(self.gitdir):
for root, dirs, files in os.walk(self.gitdir, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(self.gitdir)
if os.path.exists(mapfile):
os.remove(mapfile)
def get_branches(repo_path):
repo = Repo(repo_path)
return [
ref[11:] for ref in repo.get_refs() if ref.startswith('refs/heads/')
]
class DiskRefsContainerTests(RefsContainerTests, TestCase):
def setUp(self):
TestCase.setUp(self)
self._repo = open_repo("refs.git")
self.addCleanup(tear_down_repo, self._repo)
self._refs = self._repo.refs
def test_get_packed_refs(self):
self.assertEqual(
{
b"refs/heads/packed": b"42d06bd4b77fed026b154d16493e5deab78f02ec",
b"refs/tags/refs-0.1": b"df6800012397fb85c56e7418dd4eb9405dee075c",
},
self._refs.get_packed_refs(),
)
def test_get_peeled_not_packed(self):
# not packed
self.assertEqual(None, self._refs.get_peeled(b"refs/tags/refs-0.2"))
self.assertEqual(b"3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8", self._refs[b"refs/tags/refs-0.2"])
# packed, known not peelable
self.assertEqual(self._refs[b"refs/heads/packed"], self._refs.get_peeled(b"refs/heads/packed"))
# packed, peeled
self.assertEqual(b"42d06bd4b77fed026b154d16493e5deab78f02ec", self._refs.get_peeled(b"refs/tags/refs-0.1"))
def test_setitem(self):
RefsContainerTests.test_setitem(self)
f = open(os.path.join(self._refs.path, "refs", "some", "ref"), "rb")
self.assertEqual(b"42d06bd4b77fed026b154d16493e5deab78f02ec", f.read()[:40])
f.close()
def test_setitem_symbolic(self):
ones = b"1" * 40
self._refs[b"HEAD"] = ones
self.assertEqual(ones, self._refs[b"HEAD"])
# ensure HEAD was not modified
f = open(os.path.join(self._refs.path, "HEAD"), "rb")
self.assertEqual(b"ref: refs/heads/master", next(iter(f)).rstrip(b"\n"))
f.close()
# ensure the symbolic link was written through
f = open(os.path.join(self._refs.path, "refs", "heads", "master"), "rb")
self.assertEqual(ones, f.read()[:40])
f.close()
def test_set_if_equals(self):
RefsContainerTests.test_set_if_equals(self)
# ensure symref was followed
self.assertEqual(b"9" * 40, self._refs[b"refs/heads/master"])
# ensure lockfile was deleted
self.assertFalse(os.path.exists(os.path.join(self._refs.path, "refs", "heads", "master.lock")))
self.assertFalse(os.path.exists(os.path.join(self._refs.path, "HEAD.lock")))
def test_add_if_new_packed(self):
# don't overwrite packed ref
self.assertFalse(self._refs.add_if_new(b"refs/tags/refs-0.1", b"9" * 40))
self.assertEqual(b"df6800012397fb85c56e7418dd4eb9405dee075c", self._refs[b"refs/tags/refs-0.1"])
def test_add_if_new_symbolic(self):
# Use an empty repo instead of the default.
repo_dir = os.path.join(tempfile.mkdtemp(), "test")
os.makedirs(repo_dir)
repo = Repo.init(repo_dir)
self.addCleanup(tear_down_repo, repo)
refs = repo.refs
nines = b"9" * 40
self.assertEqual(b"ref: refs/heads/master", refs.read_ref(b"HEAD"))
self.assertFalse(b"refs/heads/master" in refs)
self.assertTrue(refs.add_if_new(b"HEAD", nines))
self.assertEqual(b"ref: refs/heads/master", refs.read_ref(b"HEAD"))
self.assertEqual(nines, refs[b"HEAD"])
self.assertEqual(nines, refs[b"refs/heads/master"])
self.assertFalse(refs.add_if_new(b"HEAD", b"1" * 40))
self.assertEqual(nines, refs[b"HEAD"])
self.assertEqual(nines, refs[b"refs/heads/master"])
def test_follow(self):
self.assertEqual(
(b"refs/heads/master", b"42d06bd4b77fed026b154d16493e5deab78f02ec"), self._refs._follow(b"HEAD")
)
self.assertEqual(
(b"refs/heads/master", b"42d06bd4b77fed026b154d16493e5deab78f02ec"),
self._refs._follow(b"refs/heads/master"),
)
self.assertRaises(KeyError, self._refs._follow, b"refs/heads/loop")
def test_delitem(self):
RefsContainerTests.test_delitem(self)
ref_file = os.path.join(self._refs.path, "refs", "heads", "master")
self.assertFalse(os.path.exists(ref_file))
self.assertFalse(b"refs/heads/master" in self._refs.get_packed_refs())
def test_delitem_symbolic(self):
self.assertEqual(b"ref: refs/heads/master", self._refs.read_loose_ref(b"HEAD"))
del self._refs[b"HEAD"]
self.assertRaises(KeyError, lambda: self._refs[b"HEAD"])
self.assertEqual(b"42d06bd4b77fed026b154d16493e5deab78f02ec", self._refs[b"refs/heads/master"])
self.assertFalse(os.path.exists(os.path.join(self._refs.path, "HEAD")))
def test_remove_if_equals_symref(self):
# HEAD is a symref, so shouldn't equal its dereferenced value
self.assertFalse(self._refs.remove_if_equals(b"HEAD", b"42d06bd4b77fed026b154d16493e5deab78f02ec"))
self.assertTrue(self._refs.remove_if_equals(b"refs/heads/master", b"42d06bd4b77fed026b154d16493e5deab78f02ec"))
self.assertRaises(KeyError, lambda: self._refs[b"refs/heads/master"])
# HEAD is now a broken symref
self.assertRaises(KeyError, lambda: self._refs[b"HEAD"])
self.assertEqual(b"ref: refs/heads/master", self._refs.read_loose_ref(b"HEAD"))
self.assertFalse(os.path.exists(os.path.join(self._refs.path, "refs", "heads", "master.lock")))
self.assertFalse(os.path.exists(os.path.join(self._refs.path, "HEAD.lock")))
def test_remove_packed_without_peeled(self):
refs_file = os.path.join(self._repo.path, "packed-refs")
f = GitFile(refs_file)
refs_data = f.read()
f.close()
f = GitFile(refs_file, "wb")
f.write(b"\n".join(l for l in refs_data.split(b"\n") if not l or l[0] not in b"#^"))
f.close()
self._repo = Repo(self._repo.path)
refs = self._repo.refs
self.assertTrue(refs.remove_if_equals(b"refs/heads/packed", b"42d06bd4b77fed026b154d16493e5deab78f02ec"))
def test_remove_if_equals_packed(self):
# test removing ref that is only packed
self.assertEqual(b"df6800012397fb85c56e7418dd4eb9405dee075c", self._refs[b"refs/tags/refs-0.1"])
self.assertTrue(self._refs.remove_if_equals(b"refs/tags/refs-0.1", b"df6800012397fb85c56e7418dd4eb9405dee075c"))
self.assertRaises(KeyError, lambda: self._refs[b"refs/tags/refs-0.1"])
def test_read_ref(self):
self.assertEqual(b"ref: refs/heads/master", self._refs.read_ref(b"HEAD"))
self.assertEqual(b"42d06bd4b77fed026b154d16493e5deab78f02ec", self._refs.read_ref(b"refs/heads/packed"))
self.assertEqual(None, self._refs.read_ref(b"nonexistant"))
def test_non_ascii(self):
try:
encoded_ref = u"refs/tags/schön".encode(sys.getfilesystemencoding())
except UnicodeDecodeError:
raise SkipTest("filesystem encoding doesn't support special character")
p = os.path.join(self._repo.path, "refs", "tags", "schön")
with open(p, "w") as f:
f.write("00" * 20)
expected_refs = dict(_TEST_REFS)
expected_refs[encoded_ref] = b"00" * 20
self.assertEqual(expected_refs, self._repo.get_refs())
