def classify_by_tag(
path: str, start_tag: str, end_tag: Optional[str] = None, model: Optional[MLModel] = None
) -> List[str]:
"""Classify messages for the given repo based on tags."""
repo_path = os.path.join(path, ".git")
if os.path.exists(repo_path):
repo = Repository(repo_path)
else:
raise RepositoryNotFoundException
start_tag = repo.revparse_single("refs/tags/" + start_tag)
if end_tag is None:
end_tag = repo.revparse_single("refs/heads/master")
else:
end_tag = repo.revparse_single("refs/tags/" + end_tag)
orig_messages = []
walker = repo.walk(end_tag.id, GIT_SORT_TOPOLOGICAL)
walker.hide(start_tag.id)
for commit in walker:
orig_messages.append(commit.message.lower())
return classify_messages(orig_messages, model)
def main(args):
repo = Repository(os.getcwd())
start_commit = repo.revparse_single(getattr(args, "start-commit"))
end_commit = repo.revparse_single(getattr(args, "end-commit"))
formatter = getattr(args, "formatter")
for commit, heading, changes in get_changes_for_commits(
repo, start_commit, end_commit, formatter):
print(f"{commit}: {heading}")
print(indent(changes))
def save_git_info(path):
repo = Repository(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../.."))
info = {
"branch": repo.head.shorthand,
"commit_hex": repo.revparse_single('HEAD').hex,
"commit_message": repo.revparse_single('HEAD').message,
}
with open(path, 'w') as f:
json.dump(info, f)
def test_commit_to_head(git_repo: pygit2.Repository, tmp_path: Path):
commit = git_repo.revparse_single('HEAD~3')
call_args = [
'git-to-freshports-xml', f'--path={git_repo.workdir}',
f'--commit={commit.hex}', f'--output={tmp_path}',
f'--spool={tmp_path}', '--repo=ports'
]
with patch.object(sys, 'argv', call_args):
MODULE_UNDER_TEST.main()
result_files = list(tmp_path.iterdir())
assert len(result_files) == 3, "Must be exactly three commits"
assert git_repo.revparse_single('HEAD~2').hex in result_files[0].name
def sync_handler(fork_from: str, from_sha: str, repo_name: str,
ticket_id: int, pr_url: str):
output_path = '{}.txt'.format(pr_url.split('/', 3)[3].rsplit('/', 2)[0])
output_path = os.path.join(WORK_DIR, output_path.replace('/', '_'))
work_tree = os.path.join(WORK_DIR, fork_from)
parent_path = os.path.dirname(work_tree)
if not os.path.exists(parent_path):
os.makedirs(parent_path)
if not os.path.exists(work_tree):
repo = clone_repository(
'{0}{1}.git'.format(GITHUB_URL, fork_from), work_tree)
else:
repo = Repository(work_tree)
remote_name = repo_name.split('/')[0]
update_remote(work_tree, repo, repo_name, remote_name)
if remote_name == 'origin':
commit = repo.revparse_single(from_sha)
repo.checkout_tree(commit, strategy=GIT_CHECKOUT_FORCE)
else:
ref_name = 'refs/pull/{0}/head'.format(ticket_id)
try:
repo.create_reference(ref_name, from_sha)
except ValueError:
pass
ref = repo.lookup_reference(ref_name)
repo.checkout(ref, strategy=GIT_CHECKOUT_FORCE)
cwd = os.getcwd()
os.chdir(work_tree)
subprocess.call(
'{} . --output-file={}'.format(FLAKE8_EXECUTABLE, output_path),
shell=True)
os.chdir(cwd)
return output_path
def get_docset_ids(config: Dict, manifest: Manifest,
only: Optional[List[str]]) -> Dict[str, str]:
"""Obtain the ids for all requested docsets.
Args:
config: Cache configuration.
manifest: Repository manifest.
only: Only consider this docset (null to consider all)
Returns:
State.
"""
ids = dict()
for entry in config:
if only and entry["docset"] not in only:
continue
id = ""
for name in entry["projects"]:
p = next((p for p in manifest.projects if p.name == name), None)
assert p, f"Project {name} not in manifest"
repo = Repository(Path(p.topdir) / p.path)
id += repo.revparse_single("HEAD").id.hex
ids[entry["docset"]] = hashlib.sha256(id.encode("utf-8")).hexdigest()
return ids
def new_branch(working_dir, branch_name):
repo = Repository(working_dir + '.git')
commit = repo.revparse_single('HEAD')
print working_dir
print 'sssssssssssssssssssss'
new_branch = repo.branches.create(branch_name, commit, force=False)
print('new branch ' + branch_name)
def test_blame_mapper(gitrepo):
directory, g = gitrepo
repo = Repository("%s/jool/.git" % directory)
commit = repo.revparse_single('9aeaac94cda9eb7e32d6f861079ab793a0311983')
mapper = BugMap()
assert mapper.map(repo, 'c/d.py', 5,
commit) == '903b2d2c65f291492f30467edc6442b74a78ab92'
def get_new_articles():
blog = PelicanBlog()
content_dir = blog.get_content_directory()
repo = Repository(os.path.abspath(os.path.dirname(__file__)))
diff = repo.revparse_single("HEAD").tree.diff_to_tree()
existing_articles = set(os.path.relpath(obj.old_file_path, content_dir)
for obj in diff
if obj.old_file_path.startswith(content_dir))
all_articles = set(blog.get_posts())
new_articles = {art for art in all_articles - existing_articles
if blog.get_post_lang(art) in (TWITTER_LANGUAGE, "")}
new_titles = []
repo.index.read()
for newart in new_articles:
title = blog.get_post_title(newart)
yield Article(title, blog.get_post_url(newart),
blog.get_post_authors(newart))
new_titles.append(title)
repo.index.add(os.path.join(content_dir, newart))
blogger = Signature(repo.config["user.name"], repo.config["user.email"])
repo.create_commit("HEAD", blogger, blogger,
"[BLOG] %s" % ", ".join(new_titles),
repo.index.write_tree(), [repo.head.peel().oid])
repo.index.write()
# TODO(v.markovtsev): implement git push using pygit2
subprocess.call(("git", "push", "origin", repo.head.shorthand))
def get_docset_props(docset: str, config: Dict, manifest: Manifest) -> Tuple[str, bool]:
"""Obtain the id and dirty status of the given docset.
Args:
docset: Docset name.
config: Cache configuration.
manifest: Repository manifest.
Returns:
Docset ID.
"""
for entry in config:
if entry["docset"] != docset:
continue
id = ""
dirty = False
for name in entry["projects"]:
p = next((p for p in manifest.projects if p.name == name), None)
assert p, f"Project {name} not in manifest"
repo = Repository(Path(p.topdir) / p.path)
id += repo.revparse_single("HEAD").id.hex
dirty = dirty or bool(repo.status())
return hashlib.sha256(id.encode("utf-8")).hexdigest(), dirty
raise ValueError(f"Docset {docset} not in configuration file")
def commit_range(repo: pygit2.Repository, commit_range: str):
start_commit_ref, end_commit_ref = commit_range.split('..')
start_commit = repo.revparse_single(start_commit_ref)
end_commit = repo.revparse_single(end_commit_ref)
walker = repo.walk(end_commit.oid)
walker.simplify_first_parent(
) # Avoid wandering off to merged branches. Same as 'git log --first-parent'
result = []
for commit in walker:
if commit == start_commit:
break
result.append(commit)
return list(reversed(result))
def add_version_commit():
repo = Repository(script_dir)
create_commit(repo, 'Update to {}'.format(ver_str))
config = Config.get_global_config()
author = Signature(config['user.name'], config['user.email'])
repo.create_tag('v{}'.format(ver_str),
repo.revparse_single('HEAD').id, GIT_OBJ_COMMIT, author,
'v{}'.format(ver_str))
def makeInstanceBranch(repo: Repository, name: str) -> None:
# headCommit = repo.revparse_single('origin/master')
headCommit = repo.revparse_single('HEAD')
# branch = repo.creare_branch(name, headCommit, force = True)
print('creating branch ' + name)
branch = repo.branches.local.create(name, headCommit, force=True)
print('checkouting to ' + name)
repo.checkout(branch)
def file_at_revision(repo: pygit2.Repository, filepath: str, revision: str):
tree = repo.revparse_single(revision).tree
dirname = os.path.dirname(filepath)
if dirname != "":
for component in dirname.split("/"):
tree = repo[tree[component].oid]
return repo[tree[os.path.basename(filepath)].id].data.decode("utf-8")
def new_switch_branch(working_dir, branch_name):
repo = Repository(working_dir + '.git')
commit = repo.revparse_single('HEAD')
new_branch = repo.branches.create(branch_name, commit, force=False)
print 'refs/heads/' + new_branch.branch_name
print working_dir
print 'ddddddddddddddddd'
repo.checkout('refs/heads/' + new_branch.branch_name)
print('switch to a new branch: ' + branch_name)
def _pygit2_commits(commit, repository):
from pygit2 import Repository, GIT_SORT_TOPOLOGICAL
g = Repository(repository)
if '..' in commit:
tail, head = commit.split('..', 2)
head = head or 'HEAD'
else:
head = commit
tail = commit + '^'
walker = g.walk(g.revparse_single(head).oid, GIT_SORT_TOPOLOGICAL)
try:
walker.hide(g.revparse_single(tail).oid)
except KeyError:
pass
return walker
def test_commit_range(git_repo: pygit2.Repository, tmp_path: Path):
first_commit = git_repo.revparse_single('HEAD~3')
second_commit = git_repo.revparse_single('HEAD~1')
call_args = [
'git-to-freshports-xml', f'--path={git_repo.workdir}',
f'--commit-range={first_commit.hex}..{second_commit.hex}',
f'--output={tmp_path}', f'--spool={tmp_path}', '--repo=ports'
]
with patch.object(sys, 'argv', call_args):
MODULE_UNDER_TEST.main()
result_files = list(tmp_path.iterdir())
assert len(result_files) == 2, "Must be exactly two commits"
commit_filenames = [x.name for x in result_files]
assert git_repo.revparse_single('HEAD~2').hex in commit_filenames[0]
assert second_commit.hex in commit_filenames[1]
def get_git_description():
try:
from pygit2 import Repository
try:
repo = Repository('.')
return "'{0}_{1}'".format(repo.head.shorthand, repo.revparse_single('HEAD').short_id)
except:
return 'No_.git_dir'
except ImportError:
return 'pygit2_not_installed'
def getCommitInfo(obj, commid_id):
repo = Repository(obj.vcsDir)
commit = repo.revparse_single(commid_id)
msg = commit.message
return {
"epoch": commit.author.time,
"author": f"{commit.author.name} <{commit.author.email}>",
"shortHash": commit.id.hex[:8], # or commit.short_id.hex
"summary": msg.split("\n")[0],
"description": msg,
}
def getShortStat(obj, prevId, thisId):
"""
returns (files_changed, insertions, deletions)
"""
repo = Repository(obj.vcsDir)
diff = repo.diff(
a=repo.revparse_single(prevId).id.hex,
b=repo.revparse_single(thisId).id.hex,
)
stats = diff.stats
return (stats.files_changed, stats.insertions, stats.deletions)
def revparse(repo: git.Repository, revision: str) -> git.Oid:
try:
return repo.revparse_single(revision).oid
except KeyError as e:
fatal(f"Commit not found in {repo.path}: {e}")
except ValueError as e:
fatal(f"Bad revision: {e}")
except Exception as e:
fatal(f"Unexpected error: {type(e).__name__}: {e}")
def getCommitShortStat(obj, commit_id):
"""
returns (files_changed, insertions, deletions)
"""
repo = Repository(obj.vcsDir)
commit = repo.revparse_single(commit_id)
diff = repo.diff(
a=commit.parent_ids[0].hex,
b=commit.id.hex,
)
stats = diff.stats
return (stats.files_changed, stats.insertions, stats.deletions)
def get_current_timestamp(path_to_repository):
"""Utility method for getting the timestamp of the last commit from a Git repository.
Args:
path_to_repository (str): Path to the Git repository
Returns:
str: The timestamp of the last commit in the provided repository.
"""
repo = Repository(path_to_repository)
head = repo.revparse_single('HEAD')
return head.commit_time
def _fast_forward(self, local_path, merge_target, branch):
# fast-forward all the branches.
# pygit2 repo
repo = Repository(discover_repository(local_path))
# convert merge_target from hex into oid.
fetch_head = repo.revparse_single(merge_target)
# try to resolve a common anscestor between fetched and local
try:
head = repo.revparse_single(branch)
except KeyError:
# Doesn't exist. Create and done.
repo.create_reference(branch, fetch_head.oid)
return True, 'Created new branch: %s' % branch
if head.oid == fetch_head.oid:
return True, 'Source and target are identical.'
# raises KeyError if no merge bases found.
common_oid = repo.merge_base(head.oid, fetch_head.oid)
# Three different outcomes between the remaining cases.
if common_oid.hex not in (head.oid.hex, fetch_head.oid.hex):
# common ancestor is beyond both of these, not going to
# attempt a merge here and will assume this:
return False, 'Branch will diverge.'
elif common_oid.hex == fetch_head.oid.hex:
# Remote is also the common ancestor, so nothing to do.
return True, 'No new changes found.'
# This case remains: common_oid.hex == head.oid.hex, meaning
# this local repository is the ancestor of further changes
# fetched from the remote - remote newer, so fast-forward.
ref = repo.lookup_reference(branch)
ref.delete()
repo.create_reference(branch, fetch_head.oid)
return True, 'Fast-forwarded branch: %s' % branch
class GitObjectRenderer:
def __init__(self, reponame, cache, revspec, path):
self.repo = Repository(reponame)
self.revspec = revspec
self.path = path
self.object_map = {
Blob : self._render_blob,
Commit : self._render_commit,
Tag : self._render_tag
}
self.cache = cache
def _render_blob(self, rev):
data = rev.data.decode('utf-8', errors='ignore')
formatter = HtmlFormatter(linenos='inline')
lexer = guess_lexer(data)
blob = highlight(data, lexer, formatter)
return render_template("objects/blob.html",
repo = self.repo,
revspec = 'master', blob = blob, path = self.path)
def _render_commit(self, rev):
entries = self.repo[rev.tree[self.path].oid] if self.path else rev.tree
cached = []
for entry in entries:
cache_id = "{0}-commit-{1}".format(self.repo.get_name(), entry.hex)
hit = self.cache.get(cache_id)
if hit is not None:
cached.append((entry, hit))
else:
cached.append((entry, None))
files = sorted(cached, key=lambda x: x[0].filemode)
return render_template("objects/tree.html",
repo = self.repo,
revspec = self.revspec, rev = rev, files = files,
path = self.path)
def _render_tag(self, rev):
obj = self.repo[rev.target]
url = url_for('obj', reponame=self.repo.get_name(), revspec = obj.hex)
return redirect(url)
def render_obj(self):
rev = self.repo.revparse_single(self.revspec)
return self.object_map[type(rev)](rev)
def revparse(repo: pygit2.Repository, target_id: str, *, strict: bool = True):
assert isinstance(repo, pygit2.Repository)
try:
ref = repo.revparse_single(target_id)
short_id = ref.short_id
message = getattr(ref, 'message', None)
except (KeyError, pygit2.GitError, pygit2.InvalidSpecError):
if strict:
raise
else:
# Use given target id and an unknown message
short_id = target_id
message = None
if message is None or not message or message.isspace():
if strict:
raise ValueError(f"Invalid message for {short_id}: {message!r}")
else:
message = "<UNKNOWN MESSGAE>"
# Strip everything before the first line (giving us the summary)
if '\n' in message:
summary = message[:message.index('\n')]
return DevCommit(short_id, summary, message)
def revision_file(
repository: pygit2.Repository, revision: Optional[str], filepath: str
) -> Optional[str]:
"""
Returns the source from the file at the given filepath on the given revision. If revision is
None, returns the bytes from the filepath in the current working tree.
Filepath is expected to be an absolute, normalized path.
"""
repo_path = os.path.normpath(repository.workdir)
content = bytes()
if revision is None:
assert (
os.path.commonpath([repo_path, filepath]) == repo_path
), f"File ({filepath}) is not contained in repository ({repo_path})"
with open(filepath, "rb") as ifp:
content = ifp.read()
else:
relative_path = os.path.relpath(filepath, repo_path)
dirname, basename = os.path.split(relative_path)
components: List[str] = [basename]
while dirname:
dirname, basename = os.path.split(dirname)
components.append(basename)
components.reverse()
commit = repository.revparse_single(revision)
current_tree = commit.tree
for component in components:
try:
current_tree = current_tree[component]
except KeyError:
return None
content = current_tree.data
return content.decode(errors="ignore")
class GitRepo:
"""A class that manages a git repository.
This class enables versiong via git for a repository.
You can stage and commit files and checkout different commits of the repository.
"""
path = ''
pathspec = []
repo = None
callback = None
author_name = 'QuitStore'
author_email = '*****@*****.**'
gcProcess = None
def __init__(self, path, origin=None, gc=False):
"""Initialize a new repository from an existing directory.
Args:
path: A string containing the path to the repository.
origin: The remote URL where to clone and fetch from and push to
"""
logger = logging.getLogger('quit.core.GitRepo')
logger.debug('GitRepo, init, Create an instance of GitStore')
self.path = path
self.gc = gc
if not exists(path):
try:
makedirs(path)
except OSError as e:
raise Exception('Can\'t create path in filesystem:', path, e)
try:
self.repo = Repository(path)
except KeyError:
pass
except AttributeError:
pass
if origin:
self.callback = QuitRemoteCallbacks()
if self.repo:
if self.repo.is_bare:
raise QuitGitRepoError('Bare repositories not supported, yet')
if origin:
# set remote
self.addRemote('origin', origin)
else:
if origin:
# clone
self.repo = self.cloneRepository(origin, path, self.callback)
else:
self.repo = init_repository(path=path, bare=False)
def cloneRepository(self, origin, path, callback):
try:
repo = clone_repository(url=origin,
path=path,
bare=False,
callbacks=callback)
return repo
except Exception as e:
raise QuitGitRepoError(
"Could not clone from: {} origin. {}".format(origin, e))
def addall(self):
"""Add all (newly created|changed) files to index."""
self.repo.index.read()
self.repo.index.add_all(self.pathspec)
self.repo.index.write()
def addfile(self, filename):
"""Add a file to the index.
Args:
filename: A string containing the path to the file.
"""
index = self.repo.index
index.read()
try:
index.add(filename)
index.write()
except Exception as e:
logger.info(
"GitRepo, addfile, Could not add file {}.".format(filename))
logger.debug(e)
def addRemote(self, name, url):
"""Add a remote.
Args:
name: A string containing the name of the remote.
url: A string containing the url to the remote.
"""
try:
self.repo.remotes.create(name, url)
logger.info("Successfully added remote: {} - {}".format(name, url))
except Exception as e:
logger.info("Could not add remote: {} - {}".format(name, url))
logger.debug(e)
try:
self.repo.remotes.set_push_url(name, url)
self.repo.remotes.set_url(name, url)
except Exception as e:
logger.info("Could not set push/fetch urls: {} - {}".format(
name, url))
logger.debug(e)
def checkout(self, commitid):
"""Checkout a commit by a commit id.
Args:
commitid: A string cotaining a commitid.
"""
try:
commit = self.repo.revparse_single(commitid)
self.repo.set_head(commit.oid)
self.repo.reset(commit.oid, GIT_RESET_HARD)
logger.info("Checked out commit: {}".format(commitid))
except Exception as e:
logger.info("Could not check out commit: {}".format(commitid))
logger.debug(e)
def commit(self, message=None):
"""Commit staged files.
Args:
message: A string for the commit message.
Raises:
Exception: If no files in staging area.
"""
if self.isstagingareaclean():
# nothing to commit
return
index = self.repo.index
index.read()
tree = index.write_tree()
try:
author = Signature(self.author_name, self.author_email)
comitter = Signature(self.author_name, self.author_email)
if len(self.repo.listall_reference_objects()) == 0:
# Initial Commit
if message is None:
message = 'Initial Commit from QuitStore'
self.repo.create_commit('HEAD', author, comitter, message,
tree, [])
else:
if message is None:
message = 'New Commit from QuitStore'
self.repo.create_commit('HEAD', author, comitter, message,
tree,
[self.repo.head.get_object().hex])
logger.info('Updates commited')
except Exception as e:
logger.info('Nothing to commit')
logger.debug(e)
if self.gc:
self.garbagecollection()
def commitexists(self, commitid):
"""Check if a commit id is part of the repository history.
Args:
commitid: String of a Git commit id.
Returns:
True, if commitid is part of commit log
False, else.
"""
if commitid in self.getids():
return True
else:
return False
def garbagecollection(self):
"""Start garbage collection.
Args:
commitid: A string cotaining a commitid.
"""
try:
# Check if the garbage collection process is still running
if self.gcProcess is None or self.gcProcess.poll() is not None:
# Start garbage collection with "--auto" option,
# which imidietly terminates, if it is not necessary
self.gcProcess = Popen(["git", "gc", "--auto", "--quiet"],
cwd=self.path)
logger.debug('Spawn garbage collection')
except Exception as e:
logger.debug('Git garbage collection failed to spawn')
logger.debug(e)
def getpath(self):
"""Return the path of the git repository.
Returns:
A string containing the path to the directory of git repo
"""
return self.path
def getcommits(self):
"""Return meta data about exitsting commits.
Returns:
A list containing dictionaries with commit meta data
"""
commits = []
if len(self.repo.listall_reference_objects()) > 0:
for commit in self.repo.walk(self.repo.head.target,
GIT_SORT_REVERSE):
commits.append({
'id':
str(commit.oid),
'message':
str(commit.message),
'commit_date':
datetime.fromtimestamp(
commit.commit_time).strftime('%Y-%m-%dT%H:%M:%SZ'),
'author_name':
commit.author.name,
'author_email':
commit.author.email,
'parents': [c.hex for c in commit.parents],
})
return commits
def getids(self):
"""Return meta data about exitsting commits.
Returns:
A list containing dictionaries with commit meta data
"""
ids = []
if len(self.repo.listall_reference_objects()) > 0:
for commit in self.repo.walk(self.repo.head.target,
GIT_SORT_REVERSE):
ids.append(str(commit.oid))
return ids
def isgarbagecollectionon(self):
"""Return if gc is activated or not.
Returns:
True, if activated
False, if not
"""
return self.gc
def isstagingareaclean(self):
"""Check if staging area is clean.
Returns:
True, if staginarea is clean
False, else.
"""
status = self.repo.status()
for filepath, flags in status.items():
if flags != GIT_STATUS_CURRENT:
return False
return True
def pull(self, remote='origin', branch='master'):
"""Pull if possible.
Return:
True: If successful.
False: If merge not possible or no updates from remote.
"""
try:
self.repo.remotes[remote].fetch()
except Exception as e:
logger.info("Can not pull: Remote {} not found.".format(remote))
logger.debug(e)
ref = 'refs/remotes/' + remote + '/' + branch
remoteid = self.repo.lookup_reference(ref).target
analysis, _ = self.repo.merge_analysis(remoteid)
if analysis & GIT_MERGE_ANALYSIS_UP_TO_DATE:
# Already up-to-date
pass
elif analysis & GIT_MERGE_ANALYSIS_FASTFORWARD:
# fastforward
self.repo.checkout_tree(self.repo.get(remoteid))
master_ref = self.repo.lookup_reference('refs/heads/master')
master_ref.set_target(remoteid)
self.repo.head.set_target(remoteid)
elif analysis & GIT_MERGE_ANALYSIS_NORMAL:
self.repo.merge(remoteid)
tree = self.repo.index.write_tree()
msg = 'Merge from ' + remote + ' ' + branch
author = Signature(self.author_name, self.author_email)
comitter = Signature(self.author_name, self.author_email)
self.repo.create_commit('HEAD', author, comitter, msg, tree,
[self.repo.head.target, remoteid])
self.repo.state_cleanup()
else:
logger.debug('Can not pull. Unknown merge analysis result')
def push(self, remote='origin', branch='master'):
"""Push if possible.
Return:
True: If successful.
False: If diverged or nothing to push.
"""
ref = ['refs/heads/' + branch]
try:
remo = self.repo.remotes[remote]
except Exception as e:
logger.info(
"Can not push. Remote: {} does not exist.".format(remote))
logger.debug(e)
return
try:
remo.push(ref, callbacks=self.callback)
except Exception as e:
logger.info("Can not push to {} with ref {}".format(
remote, str(ref)))
logger.debug(e)
def getRemotes(self):
remotes = {}
try:
for remote in self.repo.remotes:
remotes[remote.name] = [remote.url, remote.push_url]
except Exception as e:
logger.info('No remotes found.')
logger.debug(e)
return {}
return remotes
class PyGitEngine(GitContentDatabaseEngine):
def __init__(self, config):
super(PyGitEngine, self).__init__(config)
self.repo = None
def connect(self):
"""Create content directory"""
if not isdir(self.content_path):
init_repository(self.content_path, bare=True)
self.repo = Repository(self.content_path)
self.create_initial_commit()
else:
self.repo = Repository(self.content_path)
@staticmethod
def do_put(content_path, object_hashes, content, filename):
"""Perform put operation. This is used in the distributed wrapper"""
content_hash = Repository(content_path).create_blob(content)
result = object_hashes[filename] = str(content_hash)
return result
def put_attr(self, content, filename):
"""Return attributes for the do_put operation"""
filename = self._inc_name(filename)
return (self.content_path, self.object_hashes, content, filename)
def put(self, content, filename="generic"): # pylint: disable=method-hidden
"""Put content in the content database"""
return self.do_put(*self.put_attr(content, filename))
def get(self, content_hash): # pylint: disable=method-hidden
"""Get content from the content database"""
return_data = self.repo[content_hash].data
return return_data
def find_subhash(self, content_hash):
"""Find hash in git"""
try:
blob = self.repo.revparse_single(content_hash)
return str(blob.id)
except KeyError:
return None
def create_initial_commit(self):
"""Create the initial commit of the git repository"""
empty_tree = self.repo.TreeBuilder().write()
self.create_commit_object(self._initial_message, empty_tree)
def create_commit_object(self, message, tree):
"""Create a commit object"""
references = list(self.repo.references)
master_ref = self.repo.lookup_reference(
self._commit_ref) if len(references) > 0 else None
parents = []
if master_ref is not None:
parents = [master_ref.peel().id]
author = Signature(self._commit_name, self._commit_email)
return self.repo.create_commit(self._commit_ref, author, author,
message, tree, parents)
def new_tree(self, parent):
"""Create new git tree"""
return self.repo.TreeBuilder()
def insert_blob(self, tree, basename, value):
"""Insert blob into tree"""
tree.insert(basename, value, GIT_FILEMODE_BLOB)
def insert_tree(self, tree, basename, value):
"""Insert tree into tree"""
tree.insert(basename, value, GIT_FILEMODE_TREE)
def write_tree(self, tree):
"""Write tree to git directory"""
return tree.write()
def main():
git_repo = Path(sys.argv[1]).resolve(strict=True)
modules_dir = Path(sys.argv[2]).resolve(strict=True)
collections_dir = Path(sys.argv[3]).resolve(strict=True)
reference = sys.argv[4]
canonical_file = Path(sys.argv[5]).resolve(strict=True)
repo = Repository(git_repo)
canonical_list = json.load(canonical_file.open())
canonical_mapping = {}
for bookslug in reversed(canonical_list):
collection = collections_dir / f'{bookslug}.collection.xml'
col_tree = etree.parse(str(collection))
col_modules = col_tree.xpath("//col:module/@document",
namespaces={"col": NS_COLLXML})
col_uuid = col_tree.xpath("//md:uuid", namespaces={"md":
NS_MDML})[0].text
for module in col_modules:
canonical_mapping[module] = col_uuid
# For the time being, we're going to parse the timestamp of the HEAD
# commit and use that as the revised time for all module pages.
commit = repo.revparse_single('HEAD')
revised_time = datetime.fromtimestamp(commit.commit_time,
timezone.utc).isoformat()
book_version = determine_book_version(reference, repo, commit)
# Get list of module files while filtering orphans using canonical_mapping
module_files = [
mf.resolve(strict=True) for mf in modules_dir.glob("**/*")
if mf.is_file() and mf.name == "index.cnxml"
and mf.parent.name in canonical_mapping
]
collection_files = [
cf.resolve(strict=True) for cf in collections_dir.glob("*.xml")
]
for module_file in module_files:
cnxml_doc = etree.parse(str(module_file))
remove_metadata_entries(cnxml_doc, ["canonical-book-uuid"], NS_CNXML)
new_metadata = {
"revised": revised_time,
"canonical-book-uuid": canonical_mapping[module_file.parent.name]
}
add_metadata_entries(cnxml_doc, new_metadata, NS_CNXML)
with open(module_file, "wb") as f:
cnxml_doc.write(f, encoding="utf-8", xml_declaration=False)
for collection_file in collection_files:
collection_doc = etree.parse(str(collection_file))
new_metadata = {"revised": revised_time, "version": book_version}
add_metadata_entries(collection_doc, new_metadata, NS_COLLXML)
with open(collection_file, "wb") as f:
collection_doc.write(f, encoding="utf-8", xml_declaration=False)
class GitRepo(object):
''' git repo class '''
def __init__(self, path):
try:
self.__repo = Repository(path)
except Exception as e:
self.__repo = None
print(e)
def get_info(self):
if not self.__repo:
return None
signature = self.__repo.default_signature
result = {
'path': self.__repo.path,
'workdir': self.__repo.workdir,
'bare': self.__repo.is_bare,
'empty': self.__repo.is_empty,
'name': signature.name,
'email': signature.email,
'time': signature.time,
'offset': signature.offset,
}
return result
def get_all_references(self):
return self.__repo.listall_references()
def get_reference(self, name):
if not self.__repo:
return None
ref = None
try:
ref = self.__repo.lookup_reference(name)
except Exception as e:
print(e)
return ref
def get_all_branches(self, branch_type=None):
if not self.__repo:
return None
if branch_type:
return self.__repo.listall_branches(branch_type)
r = self.__repo.listall_branches(GIT_BRANCH_LOCAL | GIT_BRANCH_REMOTE)
return r
def get_branch(self, name, branch_type=GIT_BRANCH_LOCAL):
if not self.__repo:
return None
return self.__repo.lookup_branch(name, branch_type)
def check_branch(self, name, branch_type=None):
if not branch_type:
if '/' in name:
branch_type = GIT_BRANCH_REMOTE
else:
branch_type = GIT_BRANCH_LOCAL
try:
result = self.get_branch(name, branch_type)
return result
except Exception as e:
print(e)
return False
def get_current_commit(self):
if not self.__repo:
return None
commit = self.__repo.revparse_single('HEAD')
return self.get_commit(commit)
def get_commit_by_branch(self, branch):
if not self.__repo:
return None
query = 'refs/'
if hasattr(branch, 'remote_name'):
query += 'remotes/'
else:
query += 'heads/'
query += branch.branch_name
try:
ref = self.get_reference(query)
commit = ref.target
return self.get_commit(commit)
except Exception as e:
print(e)
return None
def get_commit_by_tag(self, tag):
if self.__repo is None:
return None
if tag:
commit = tag.get_object()
return self.get_commit(commit)
return None
def get_commit(self, oid_or_commit):
''' return a commit w/ json '''
if not self.__repo or not oid_or_commit:
return None
try:
commit = oid_or_commit
if not isinstance(oid_or_commit, Commit):
commit = self.__repo.get(oid_or_commit)
if commit and commit.type == GIT_OBJ_COMMIT:
# t1 = self.__repo.revparse_single('HEAD^')
# t2 = self.__repo.revparse_single('HEAD^^')
# patches = self.__repo.diff(t1, t2)
# for p in patches:
# print(p.new_file_path)
result = {
'id': str(commit.id),
'author': commit.author.name,
'commiter': commit.committer.name,
'message': commit.message,
'message_encoding': commit.message_encoding,
'tree': str(commit.tree_id),
'parent': [str(pid) for pid in commit.parent_ids],
'time': str(commit.commit_time),
'time_offset': str(commit.commit_time_offset),
}
return result
except Exception as e:
print(e)
return None
def get_commits(self, depth=10, oid_or_commit=None):
result = []
if depth == 0:
return result
if oid_or_commit:
commit = self.get_commit(oid_or_commit)
else:
commit = self.get_current_commit()
if not commit:
return result
# TODO: starting from a commit or its parent
# TODO: author
result.append(commit)
depth -= 1
if commit and commit['parent']:
for parent in commit['parent']:
result.extend(self.get_commits(depth, parent))
return result
def get_commits_by_branch(self, name, path=None):
if not self.__repo:
return None
if self.check_branch(name):
ref = self.get_reference('refs/heads/' + name)
if ref:
commit = ref.target
commits = self.get_commits(commit)
result = {}
for key, val in commits.items():
if self.check_commit_by_path(val, path):
result[key] = val
return result
return None
def check_tag(self, name):
try:
ref = self.get_reference('refs/tags/' + name)
return ref
except Exception:
return False
def get_commits_by_tag(self, tag, path=None):
if not self.__repo:
return None
if tag:
commit = tag.target
commits = self.get_commits(commit)
result = {}
for key, val in commits.items():
if self.check_commit_by_path(val, path):
result[key] = val
return result
return None
def check_commit_by_path(self, commit, path):
if not commit:
return False
if path is None or len(path) == 0:
return True
result = self.get_tree(commit['tree'])
if not isinstance(path, list):
path = path.strip().split('/')
for name in path:
name = name.strip()
if name in result:
oid = result[name]
result = self.get_tree(oid)
if not result:
result = self.get_blob(oid)
return result is not None
def get_tree(self, oid, ppath=None):
if not self.__repo:
return None
try:
tree = self.__repo.get(oid)
if tree and tree.type == GIT_OBJ_TREE:
result = {}
for entry in tree:
item = {
'id': str(entry.id)
}
obj = self.__repo.get(entry.id)
if obj.type == GIT_OBJ_BLOB:
item['type'] = 'blob'
elif obj.type == GIT_OBJ_TREE:
item['type'] = 'tree'
item['ppath'] = ppath
result[entry.name] = item
return result
except Exception as e:
print(e)
return None
def get_tree_by_commit(self, commit, path=None):
if not commit:
return None
result = self.get_tree(commit['tree'])
if not path:
return result
# if not isinstance(path, list):
# path = path.strip().split('/')
try:
for name in path:
oid = result[name]['id']
p = result[name]['ppath']
p = name if not p else p + '/' + name
result = self.get_tree(oid, p)
if not result:
break
except Exception as e:
print(e)
result = None
return result
def get_current_root(self):
tree = self.get_current_commit()
if tree:
return self.get_tree(tree['tree'])
return None
def get_whole_tree(self, oid):
''' tree w/ json '''
if not self.__repo:
return None
result = tree_walker(self.__repo, oid)
return result
def get_blob(self, oid):
''' blob w/ json '''
if not self.__repo or not oid:
return None
try:
blob = self.__repo.get(oid)
if blob and blob.type == GIT_OBJ_BLOB:
content = blob.is_binary and None or blob.data.decode(
'utf8', 'ignore')
result = {
'id': str(blob.id),
'content': content,
'size': blob.size,
}
return result
except Exception as e:
print(e)
return None
def get_blob_by_commit(self, commit, path=None):
try:
tree = self.get_tree_by_commit(commit, path[:-1])
oid = tree[path[-1]]['id']
result = self.get_blob(oid)
return result
except Exception as e:
print(e)
return None
def get_tag(self, oid):
''' blob w/ json '''
if not self.__repo or not oid:
return None
try:
tag = self.__repo.get(oid)
if tag and tag.type == GIT_OBJ_TAG:
result = {
'id': str(oid),
'name': tag.name,
'target': str(tag.target.id),
'tagger': tag.tagger,
'message': tag.message,
}
return result
except Exception as e:
print(e)
return None
def get_patches(self, a=None, b=None):
try:
if not a:
a = 'HEAD'
if not b:
b = a + '^'
t1 = self.__repo.revparse_single(a)
t2 = self.__repo.revparse_single(b)
patches = self.__repo.diff(t1, t2)
result = []
for patch in patches:
p = {
'old_file_path': patch.old_file_path,
'new_file_path': patch.new_file_path,
'old_oid': str(patch.old_oid),
'new_oid': str(patch.new_oid),
'status': patch.status,
'similarity': patch.similarity,
'additions': patch.additions,
'deletions': patch.deletions,
'binary': patch.is_binary,
'hunks': [],
}
for hunk in patch.hunks:
h = {
'old_start': hunk.old_start,
'old_lines': hunk.old_lines,
'new_start': hunk.new_start,
'new_lines': hunk.new_lines,
'lines': hunk.lines,
}
p['hunks'].append(h)
result.append(p)
return result
except Exception as e:
print(e)
return None
logging.info('Checking if test-metric recording needs to be enabled')
if 'TEST_METRICS_URL' in os.environ:
import datetime
import getpass
import json
import socket
from timeit import default_timer as timer
from pygit2 import Repository
from tests.cook import util
repository_path = os.path.abspath(f'{os.path.dirname(os.path.abspath(__file__))}/../..')
repo = Repository(repository_path)
head = repo.head
commit = repo.revparse_single('HEAD')
git_branch = head.name.replace('refs/heads/', '')
git_commit_hex = commit.hex
elastic_search_url = os.getenv('TEST_METRICS_URL').rstrip('/')
logging.info(f'Sending test metrics to {elastic_search_url}')
@pytest.fixture()
def record_test_metric(request):
start = timer()
yield
try:
end = timer()
now = datetime.datetime.utcnow()
index = f'cook-tests-{now.strftime("%Y%m%d")}'
request_node = request.node
class GitOperations(object):
def __init__(self, repo, no_cache=False):
self.repo = repo
self._gitrepo = os.path.join(repo, '.git')
self._pygit = Repository(repo)
self._commands = {}
self._trees = {}
self._trees_filled = {}
self._sizes = {}
self._refs = {}
self._commits_iterator = None
self.cache = not no_cache
self.years = range(self._first_year(), self._last_year() + 1)
def cached_command(self, list, return_exit_code=False, silent=False):
"""
Executes the specified git command and returns its result.
Subsequent executions of the same command return the cached result
If return_exit_code is set, then the return value is True of False
depending on whether the command exited with 0 or not.
If silent is true then failed executions return None,
without displaying an error.
"""
list = ['git', '--git-dir', self._gitrepo] + list
command = " ".join(list)
if command in self._commands:
return self._commands[command]
else:
try:
# print(command)
out = check_output(list)
if return_exit_code:
out = True
except CalledProcessError as e:
if return_exit_code:
out = False
elif silent:
out = None
else:
message = "Error calling %s: %s" % (command, str(e))
sys.stderr.write(message)
out = None
if self.cache:
self._commands[command] = out
return out
def _get_entry(self, commit, path=None, return_tree=False):
try:
if path:
obj = self._pygit[commit].tree[path]
elif path == '':
obj = self._pygit[commit].tree
else:
obj = self._pygit[commit]
except KeyError as e:
raise GitOperError("pygit entry does not exist\n%s" % (str(e)))
if return_tree:
return obj
return self._pygit[obj.id]
def _fill_trees(self, commit, contents):
if not commit in self._trees:
self._trees[commit] = set()
trees = []
for cont in contents:
if cont[1] == "tree" and cont[0] not in self._trees[commit]:
trees.append(cont[0])
self._trees[commit].update(trees)
def _get_tree(self, commit, path):
if not path:
tree = self._get_entry(commit, '')
else:
path += "/"
try:
tree = self._get_entry(commit, path)
except KeyError:
return []
return [(c.name, c.type) for c in tree]
def _cache_tree(self, commit, path):
tree = self._get_tree(commit, path)
paths_and_names = [(os.path.join(path, c[0]), c[1]) for c in tree]
self._fill_trees(commit, paths_and_names)
if not commit in self._trees_filled:
self._trees_filled[commit] = set()
self._trees_filled[commit].update([path])
def _first_year(self):
"""
Returns the year of the repo's first commit(s)
Not implemented using pygit2 because its faster
to get the year via shell command and it creates
only one process on boot time.
"""
first_years = self.cached_command(
['log', '--max-parents=0', '--date=format:%Y',
'--pretty=%ad']).splitlines()
return int(sorted(first_years)[0])
def _last_year(self):
"""
Returns the year of the repo's last commit
"""
return int(
self.cached_command([
'log', '-n', '1', '--all', '--date=format:%Y', '--pretty=%ad'
]))
def refs(self, refs):
"""
Returns the specified refs in the form:
<commit_hash> refs/{heads,remotes,tags}/<branchname>
"""
refs = self.cached_command(
['for-each-ref', '--format=%(objectname) %(refname)'] +
refs).splitlines()
return [ref.strip() for ref in refs]
def _get_commits_iterator(self, command):
return StringIO.StringIO(self.cached_command(command))
def commits_by_date(self, y, m, d):
"""
Returns a list of commit hashes for the given year, month, day
"""
start = datetime.date(y, m, d)
end = start + datetime.timedelta(days=1)
# T00:00:00 is at the start of the specified day
command = [
'log', '--after',
'%04d-%02d-%02dT00:00:00' % (start.year, start.month, start.day),
'--before',
'%04d-%02d-%02dT00:00:00' % (end.year, end.month, end.day),
'--all', '--pretty=%H'
]
commits = self._get_commits_iterator(command)
for commit in commits:
yield commit.strip()
def all_commits(self, prefix=""):
"""
Returns a list of all commit hashes
"""
command = ['log', '--all', '--pretty=%H']
commits = self._get_commits_iterator(command)
if prefix:
commits = iter([c for c in commits if c.startswith(prefix)])
for commit in commits:
yield commit.strip()
def _get_commit_from_ref(self, ref):
commit = self._pygit.revparse_single(ref)
if isinstance(commit, Commit):
return commit
if hasattr(commit, "target"):
return self._pygit[commit.target]
return None
def commit_of_ref(self, ref):
"""
Returns the last commit of a ref.
"""
# Check cache
if ref in self._refs:
return self._refs[ref]
commit = self._get_commit_from_ref(ref)
self._refs[ref] = ""
if commit:
self._refs[ref] = str(commit.id)
return self._refs[ref]
def commit_parents(self, commit):
"""
Returns commit parents
"""
parents = self._get_entry(commit).parents
return [str(p.id) for p in parents]
def commit_descendants(self, commit):
"""
Returns commit descendants
"""
return []
def commit_names(self, commit):
"""
Returns names associated with commit
"""
return []
def get_commit_time(self, commit):
return self._get_entry(commit).commit_time
def get_author_time(self, commit):
return self._get_entry(commit).author.time
def directory_contents(self, commit, path):
"""
Returns the contents of the directory
specified by `path`
"""
tree = self._get_tree(commit, path)
return [c[0] for c in tree]
def is_symlink(self, commit, path):
# the root of the repository can't be a symlink
if not path:
return False
entry = self._get_entry(commit, path, return_tree=True)
if entry.filemode == GIT_FILEMODE_LINK:
return True
return False
def is_dir(self, commit, path):
if commit in self._trees and path in self._trees[commit]:
return True
if commit not in self._trees:
self._trees[commit] = set([''])
self._trees_filled[commit] = set([''])
self._cache_tree(commit, '')
elements = path.split("/")
for i in range(len(elements) - 1):
subpath = "/".join(elements[:i + 1])
if subpath in self._trees[
commit] and subpath not in self._trees_filled[commit]:
self._cache_tree(commit, subpath)
return path in self._trees[commit]
def file_contents(self, commit, path):
try:
return self._get_entry(commit, path).data
except KeyError:
return ""
def file_size(self, commit, path):
if not commit in self._sizes:
self._sizes[commit] = {}
if path in self._sizes[commit]:
return self._sizes[commit][path]
try:
size = self._get_entry(commit, path).size
except KeyError:
size = 0
self._sizes[commit][path] = size
return size
def author(self, commit):
return self._get_entry(commit).author.name
def author_email(self, commit):
return self._get_entry(commit).author.email
# print commit.hex
# print base.revparse_single(commit.hex).message
# print commit.commit_time
# print commit.commit_time_offset
git = sh.git.bake(_cwd='/home/heather/research/spoon-knife')
for point in history:
git.checkout(point)
# print subprocess.check_output(['ohcount', 'spoon-knife'])
git.checkout(history[0])
i = 0
while i < len(history) - 2:
t0 = base.revparse_single(history[i])
t1 = base.revparse_single(history[i+1])
diff = base.diff(t0,t1)
patches = [p for p in diff]
for patch in patches:
print 'NUM HUNKS: ' + str(len(patch.hunks))
for hunk in patch.hunks:
# print hunk.lines
totesLines = 0
totesMods = 0
for line in hunk.lines:
totesLines += 1
if line[0] == '-' or line[0] == '+':
totesMods += 1
print line
print 'TOTAL LINES: ' + str(totesLines)
class prototype:
repo = "" # Path to a given repository
name = "" # Name of a repository
base = "" # Repository as defined in pygit2
# Initialization. Clones the given repository, placing it in the current directory,
# and changes to the repository directory.
def init(self, repository):
self.repo = repository
# Use regular expressions to match the last instance of a forward slash
# followed by the name of the repository, which we wish to extract, followed
# by ".git".
m = re.search('/([^/]+).git$', repository)
if m:
self.name = m.group(1)
if not os.path.isdir(self.name):
os.system('git clone ' + self.repo) # Get the repository from GitHub
self.base = Repository(self.name)
self.base.checkout('HEAD')
# Destruction. Remove the given repository from memory.
def destroy(self):
os.system('cd ' + self.name)
os.system('rm -rf ' + self.name)
# Get total LOC by given repository.
def totalRepoLOC(self):
loc = countDirLOC(self.name)
return loc
# Get total commits by a given repository
def totalRepoCommits(self):
commits = 0
for commit in self.base.walk(self.base.head.target, GIT_SORT_TOPOLOGICAL):
commits = commits + 1
return commits
# Get a list of LOC changed per commit
def locPerCommit(self):
loc = []
oldPath = os.popen('pwd')
os.chdir(self.name)
sha1 = 0
sha2 = 0
start = 1
total = self.totalRepoCommits()
# For each commit within the repository
for commit in self.base.walk(self.base.head.target, GIT_SORT_TOPOLOGICAL):
print '\r', start, '/', total,
start += 1
# Based on the SHA, use git to show the patch for that commit
sha1 = sha2
sha2 = commit.hex
if sha1 != 0:
p = os.popen('git diff --shortstat ' + sha1 + ' ' + sha2)
line = p.readline()
# line contains "# file changed, # insertions(+), # deletions(-)
# Use regular expressions to find the number of additions and deletions.
# Additions are found after ", " and before " insertion". Deletions are
# found after "(+), " and before " deletion".
m = re.search(', (.*) insertion', line)
additions = 0
deletions = 0
if m:
additions = m.group(1)
m = re.search('\(\+\), (.*) deletion', line)
if m:
deletions = m.group(1)
# Get the total and append to array
modifications = int(additions) + int(deletions)
loc.append(modifications)
os.chdir('..')
return loc
# Get a list containing the total number of line additions and deletions (including
# whitespace and comments) contained within each hunk that was changed over t
def locPerHunk(self):
loc = []
history = []
# Get the hex number for each commit within the repository
for commit in self.base.walk(self.base.head.target, GIT_SORT_TOPOLOGICAL):
sha = commit.hex
history.append(sha)
# Compare each revision in the history of the repository with the previous rev.
i = 0
while i < len(history) - 1:
t0 = self.base.revparse_single(history[i])
t1 = self.base.revparse_single(history[i+1])
diff = self.base.diff(t0,t1)
patches = [p for p in diff]
for patch in patches:
for hunk in patch.hunks:
# Check the first character in each hunk line. Only those that have
# been modified will contain a '+' (insertion) or '-' (deletion)
totalModifications = 0
for line in hunk.lines:
if line[0] == '-' or line[0] == '+':
totalModifications +=1
loc.append(totalModifications)
i += 1
return loc
# Get the total number of lines contained within a hunk, including additions, deletions,
# and surrounding non-changed lines
def locInHunk(self):
loc = []
history = []
# Get the hex number for each commit within the repository
for commit in self.base.walk(self.base.head.target, GIT_SORT_TOPOLOGICAL):
sha = commit.hex
history.append(sha)
# Compare each revision in the history of the repository with the previous rev.
i = 0
while i < len(history) - 1:
t0 = self.base.revparse_single(history[i])
t1 = self.base.revparse_single(history[i+1])
diff = self.base.diff(t0,t1)
patches = [p for p in diff]
for patch in patches:
for hunk in patch.hunks:
totalLines = 0
for line in hunk.lines:
totalLines += 1
loc.append(totalLines)
i += 1
return loc
# Perform a diff between all commits starting from oldest to newest
# and compile temp files comprised of only modified lines.
# Run cloc on temp files to get sloc for each diff set.
def slocPerDiff(self):
# Storage for commit history hashes
history = []
# Store all slocs
slocPerDiffs = []
# Move through the system history from newest to oldest commit
for commit in self.base.walk(self.base.head.target, GIT_SORT_TOPOLOGICAL | GIT_SORT_REVERSE):
history.append(commit)
i = 0
while i < len(history) - 2:
sloc = 0
t0 = self.base.revparse_single(history[i].hex)
t1 = self.base.revparse_single(history[i+1].hex)
try:
diff = self.base.diff(t0,t1)
except ValueError:
print "Caught value error."
i += 1
continue
patches = [p for p in diff]
for patch in patches:
print patch.new_file_path
hunkfile = open("tmp", 'w')
for hunk in patch.hunks:
totesLines = 0
totesMods = 0
for line in hunk.lines:
totesLines += 1
if line[0] == '-' or line[0] == '+':
totesMods += 1
hunkfile.write(line[1])
hunkfile.close()
output = subprocess.Popen('cloc ' + patch.new_file_path + ' --by-file --csv', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
start = False
for line in output.stdout.readlines():
if line[0] == 'l':
start = True
continue
if start:
temp = line.split(',')
sloc += int(temp[4].replace('\n', ''))
retval = output.wait()
os.remove("tmp")
i += 1
slocPerDiffs.append(int(sloc))
return slocPerDiffs
# Get a list containing the number of hunks changed per commit
def hunksPerCommit(self):
hunks = []
history = []
start = 1
total = self.totalRepoCommits()
# Get the hex number for each commit within the repository
for commit in self.base.walk(self.base.head.target, GIT_SORT_TOPOLOGICAL):
sha = commit.hex
history.append(sha)
# Compare each revision in the history of the repository with the previous rev.
i = 0
while i < len(history) - 1:
print '\r', start, '/', total,
start += 1
t0 = self.base.revparse_single(history[i])
t1 = self.base.revparse_single(history[i+1])
try:
diff = self.base.diff(t0,t1)
except ValueError:
print "Caught value error."
i += 1
continue
patches = [p for p in diff]
for patch in patches:
hunks.append(len(patch.hunks))
i += 1
return hunks
# Get a list of the number of files changed per commit
def filesPerCommit(self):
files = []
oldPath = os.popen('pwd')
os.chdir(self.name)
sha1 = 0
sha2 = 0
start = 1
total = self.totalRepoCommits()
# For each commit within the repository
for commit in self.base.walk(self.base.head.target, GIT_SORT_TOPOLOGICAL):
print '\r', start, '/', total,
start += 1
# Based on the SHA, use git to show the patch for that commit
sha1 = sha2
sha2 = commit.hex
if sha1 != 0:
p = os.popen('git diff --shortstat ' + sha1 + ' ' + sha2)
line = p.readline()
# line contains "# file changed, # insertions(+), # deletions(-)
# Use regular expressions to find the number of files modified, which
# are contained first on the line followed by " file"
m = re.search(' (.*) file', line)
if m:
numFilesChanged = int(m.group(1))
files.append(numFilesChanged)
os.chdir('..')
return files
# Print out all stats for the repository
def printStats(self):
f = open(self.name + '-results.txt', 'w')
f.write(("-----------" + self.name + "-----------\n"))
# Stats on entire repository
repoLOC = self.totalRepoLOC()
repoCommits = self.totalRepoCommits()
# Lists by commit
locPerCommit = self.locPerCommit()
#slocPerDiff = self.slocPerDiff()
hunksPerCommit = self.hunksPerCommit()
filesPerCommit = self.filesPerCommit()
# Stats for LOC
xsmall = 0
small = 0
medium = 0
large = 0
xlarge = 0
for item in locPerCommit:
if (item >= 0 and item <= 5):
xsmall += 1
if (item >= 6 and item <= 46):
small += 1
if (item >= 47 and item <= 106):
medium += 1
if (item >= 107 and item <= 166):
large += 1
if (item >= 167):
xlarge += 1
f.write("Number of Modified Lines:\n")
f.write("x-small: " + str(xsmall) + "\n")
f.write("small: " + str(small) + "\n")
f.write("medium: " + str(medium) + "\n")
f.write("large: " + str(large) + "\n")
f.write("x-large: " + str(xlarge) + "\n")
'''
# Stats for SLOC
xsmall = 0
small = 0
medium = 0
large = 0
xlarge = 0
for item in slocPerDiff:
if (item >= 0 and item <= 5):
xsmall += 1
if (item >= 6 and item <= 46):
small += 1
if (item >= 47 and item <= 106):
medium += 1
if (item >= 107 and item <= 166):
large += 1
if (item >= 167):
xlarge += 1
f.write("Number of Modified SLOC: \n")
f.write("x-small: " + str(xsmall) + "\n")
f.write("small: " + str(small) + "\n")
f.write("medium: " + str(medium) + "\n")
f.write("large: " + str(large) + "\n")
f.write("x-large: " + str(xlarge) + "\n")
'''
# Print stats for modified files
xsmall = 0
small = 0
medium = 0
large = 0
xlarge = 0
for item in filesPerCommit:
if (item == 1):
xsmall += 1
if (item >= 2 and item <= 4):
small += 1
if (item >= 5 and item <= 7):
medium += 1
if (item >= 8 and item <= 10):
large += 1
if (item >= 11):
xlarge += 1
f.write("Number of modified files: \n")
f.write("x-small: " + str(xsmall) + "\n")
f.write("small: " + str(small) + "\n")
f.write("medium: " + str(medium) + "\n")
f.write("large: " + str(large) + "\n")
f.write("x-large: " + str(xlarge) + "\n")
# Prints stats for hunks
xsmall = 0
small = 0
medium = 0
large = 0
xlarge = 0
for item in hunksPerCommit:
if (item >= 0 and item <= 1):
xsmall += 1
if (item >= 2 and item <= 8):
small += 1
if (item >= 9 and item <= 17):
medium += 1
if (item >= 18 and item <= 26):
large += 1
if (item >= 27):
xlarge += 1
f.write("Number of hunks per commit: \n")
f.write("x-small: " + str(xsmall) + "\n")
f.write("small: " + str(small) + "\n")
f.write("medium: " + str(medium) + "\n")
f.write("large: " + str(large) + "\n")
f.write("x-large: " + str(xlarge) + "\n")
f.close()
class GitStorage(BaseStorage):
_backend = None
def __init__(self, context, repo_path=None):
self.context = context
rp = IStorageInfo(context).path
try:
self.repo = Repository(discover_repository(rp))
except KeyError:
# discover_repository may have failed.
raise PathNotFoundError('repository does not exist at path')
self.checkout() # defaults to HEAD.
@property
def empty_root(self):
return {'': '_empty_root'}
def _get_empty_root(self):
return self.empty_root
def _get_obj(self, path, cls=None):
if path == '' and self._commit is None:
# special case
return self._get_empty_root()
if self._commit is None:
raise PathNotFoundError('repository is empty')
root = self._commit.tree
try:
breadcrumbs = []
fragments = list(reversed(path.split('/')))
node = root
oid = None
while fragments:
fragment = fragments.pop()
if not fragment == '':
# no empty string entries, also skips over '//' and
# leaves the final node (if directory) as the tree.
oid = node[fragment].oid
node = self.repo.get(oid)
breadcrumbs.append(fragment)
if node is None:
# strange. Looks like it's either submodules only
# have entry nodes or pygit2 doesn't fully support
# this. Try to manually resolve the .gitmodules
# file.
if cls is None:
# Only return this if a specific type was not
# expected.
submods = parse_gitmodules(self.repo.get(
root[GIT_MODULE_FILE].oid).data)
submod = submods.get('/'.join(breadcrumbs))
if submod:
fragments.reverse()
return {
'': '_subrepo',
'location': submod,
'path': '/'.join(fragments),
'rev': oid.hex,
}
if node and (cls is None or isinstance(node, cls)):
return node
except KeyError:
# can't find what is needed in repo, raised by pygit2
raise PathNotFoundError('path not found')
# not what we were looking for.
if cls == Tree:
raise PathNotDirError('path not dir')
elif cls == Blob:
raise PathNotFileError('path not file')
raise PathNotFoundError('path not found')
@property
def _commit(self):
return self.__commit
@property
def rev(self):
if self.__commit:
return self.__commit.hex
return None
@property
def shortrev(self):
# TODO this is an interim solution.
if self.rev:
return self.rev[:12]
def basename(self, name):
return name.split('/')[-1]
def checkout(self, rev=None):
# None maps to the default revision.
if rev is None:
rev = 'HEAD'
try:
self.__commit = self.repo.revparse_single(rev)
except KeyError:
if rev == 'HEAD':
# probably a new repo.
self.__commit = None
return
raise RevisionNotFoundError('revision %s not found' % rev)
# otherwise a RevisionNotFoundError should be raised.
def files(self):
def _files(tree, current_path=None):
results = []
for node in tree:
if current_path:
name = '/'.join([current_path, node.name])
else:
name = node.name
obj = self.repo.get(node.oid)
if isinstance(obj, Blob):
results.append(name)
elif isinstance(obj, Tree):
results.extend(_files(obj, name))
return results
if not self._commit:
return []
results = _files(self._commit.tree)
return results
def file(self, path):
return self._get_obj(path, Blob).data
def listdir(self, path):
if path:
tree = self._get_obj(path, Tree)
else:
if self._commit is None:
return []
tree = self._commit.tree
return [entry.name for entry in tree]
def format(self, **kw):
# XXX backwards compatibility??
return kw
def log(self, start, count, branch=None, shortlog=False):
"""
start and branch are literally the same thing.
"""
def _log(iterator):
for pos, commit in iterator:
if pos == count:
raise StopIteration
yield {
'author': commit.committer.name,
'email': self._commit.committer.email,
'date': self.strftime(committer_dt(commit.committer)),
'node': commit.hex,
'rev': commit.hex,
'desc': commit.message
}
if start is None:
# assumption.
start = 'HEAD'
try:
self.repo.revparse_single(start)
except KeyError:
return []
try:
rev = self.repo.revparse_single(start).hex
except KeyError:
raise RevisionNotFoundError('revision %s not found' % start)
iterator = enumerate(self.repo.walk(rev, GIT_SORT_TIME))
return list(_log(iterator))
def pathinfo(self, path):
obj = self._get_obj(path)
if isinstance(obj, Blob):
return self.format(**{
'type': 'file',
'basename': self.basename(path),
'size': obj.size,
'date': self.strftime(committer_dt(self._commit.committer)),
})
elif isinstance(obj, dict):
# special cases are represented as dict.
if obj[''] == '_subrepo':
return self.format(**{
'type': 'subrepo',
'date': '',
'size': 0,
'basename': self.basename(path),
# extra field.
'obj': obj,
})
elif obj[''] == '_empty_root':
return self.format(**{
'type': 'folder',
'date': '',
'size': 0,
'basename': self.basename(path),
})
# Assume this is a Tree.
return self.format(**{
'basename': self.basename(path),
'size': 0,
'type': 'folder',
'date': '',
})
def branches(self):
return tuple(
(b, self.repo.lookup_branch(b).target.hex)
for b in self.repo.listall_branches()
)
def tags(self):
return tuple(
(b[10:], self.repo.lookup_reference(b).target.hex)
for b in self.repo.listall_references()
if b.startswith('refs/tags')
)
class CollectGit(object):
"""
Small Helper class for small repositories.
This does not scale because we hold a lot of data in memory.
"""
_regex_comment = re.compile(
r"(//[^\"\n\r]*(?:\"[^\"\n\r]*\"[^\"\n\r]*)*[\r\n]|/\*([^*]|\*(?!/))*?\*/)(?=[^\"]*(?:\"[^\"]*\"[^\"]*)*$)"
)
_regex_jdoc_line = re.compile(r"(- |\+)\s*(\*|/\*).*")
def __init__(self, path):
if not path.endswith('.git'):
if not path.endswith('/'):
path += '/'
path += '.git'
self._log = logging.getLogger(self.__class__.__name__)
self._path = path
self._repo = Repository(self._path)
self._hunks = {}
self._file_actions = {}
self._bugfix = {}
self._msgs = {}
self._days = {}
self._cdays = {}
self._branches = {}
self._tags = {}
self._dopts = GIT_DIFF_FIND_RENAMES | GIT_DIFF_FIND_COPIES
self._SIMILARITY_THRESHOLD = 50
self._graph = nx.DiGraph()
@classmethod
def clone_repo(cls, uri, local_path):
project_name = uri.split('/')[-1].split('.git')[0]
repo_path = local_path + '/' + project_name + '/'
if os.path.isdir(repo_path):
c = subprocess.run(['git', 'fetch'],
cwd=repo_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if c.returncode != 0:
err = 'Error pulling repository {} to {}'.format(
uri, repo_path)
raise Exception(err)
else:
os.mkdir(repo_path)
c = subprocess.run(['git', 'clone', uri, repo_path],
cwd=repo_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if c.returncode != 0:
err = 'Error cloning repository {} to {}'.format(
uri, repo_path)
raise Exception(err)
return repo_path
def _changed_lines(self, hunk):
added_lines = []
deleted_lines = []
del_line = hunk['old_start']
add_line = hunk['new_start']
for line in hunk['content'].split('\n'):
tmp = line[1:].strip()
# is_comment = tmp.startswith('//') or tmp.startswith('/*') or tmp.startswith('*')
if line.startswith('+'):
added_lines.append((add_line, tmp))
del_line -= 1
if line.startswith('-'):
deleted_lines.append((del_line, tmp))
add_line -= 1
del_line += 1
add_line += 1
return added_lines, deleted_lines
def _comment_only_change(self, content):
content = content + '\n' # required for regex to drop comments
content = re.sub(self._regex_comment, "", content)
removed = ''
added = ''
for line in content.split('\n'):
line = re.sub(
r"\s+", " ", line, flags=re.UNICODE
) # replace all kinds of whitespaces (also multiple) with sińgle whitespace
if not re.match(self._regex_jdoc_line, line):
if line.startswith('-'):
removed += line[1:].strip()
elif line.startswith('+'):
added += line[1:].strip()
return removed == added
def _blame_lines(self,
revision_hash,
filepath,
strategy,
ignore_lines=False,
validated_bugfix_lines=False):
"""We want to find changed lines for one file in one commit (from the previous commit).
For this we are iterating over the diff and counting the lines that are deleted (changed) from the original file.
We ignore all added lines.
ignore_lines is already specific to all changed hunks of the file for which blame_lines is called
"""
c = self._repo.revparse_single('{}'.format(revision_hash))
self._hunks[revision_hash] = self._get_hunks(c)
changed_lines = []
if revision_hash not in self._hunks.keys(
) or not self._hunks[revision_hash]:
return changed_lines
for h in self._hunks[revision_hash]:
if h['new_file'] != filepath:
continue
# only whitespace or comment changes in the hunk, ignore
if strategy == 'code_only' and self._comment_only_change(
h['content']):
self._log.debug(
'detected whitepace or comment only change in {} for {}'.
format(revision_hash, filepath))
continue
added, deleted = self._changed_lines(h)
for dt in deleted:
if dt not in changed_lines and dt[1]:
if strategy == 'code_only' and dt[1].startswith(
('//', '/*', '*')):
continue
# we may only want validated lines
if validated_bugfix_lines is not False:
if dt[0] not in validated_bugfix_lines:
continue
# we may ignore lines, e.g., refactorings
if ignore_lines:
ignore = False
for start_line, end_line in ignore_lines:
if start_line <= dt[0] <= end_line:
ignore = True
break
# if we hit the line in our ignore list we continue to the next
if ignore:
# self._log.warn('ignore line {} in file {} in commit {} because of refactoring detection'.format(dt[0], filepath, revision_hash))
continue
changed_lines.append(dt)
return changed_lines
def blame(self,
revision_hash,
filepath,
strategy='code_only',
ignore_lines=False,
validated_bugfix_lines=False):
"""Collect a list of commits where the given revision and file were last changed.
Uses git blame.
:param str revision_hash: Commit for which we want to collect blame commits.
:param str filepath: File for which we want to collect blame commits.
:rtype: list
:returns: A list of tuples of blame commits and the original file for the given parameters.
"""
commits = []
# - ignore if commit is not in graph
if revision_hash not in self._graph:
return []
# # - ignore package-info.java
# if strategy == 'code_only' and filepath.lower().endswith('package-info.java'):
# self._log.debug('skipping blame on revision: {} for file {} because it is package-info.java'.format(revision_hash, filepath))
# return []
# # - ignore test/ /test/ example/ examples/
# if strategy == 'code_only' and re.match(self._regex_test_example, filepath):
# self._log.debug('skipping blame on revision: {} for file {} because it is a test or an example'.format(revision_hash, filepath))
# return []
# bail on multiple parents
parents = list(self._graph.predecessors(revision_hash))
if len(parents) > 1:
self._log.debug(
'skipping blame on revision: {} because it is a merge commit'.
format(revision_hash))
return []
changed_lines = self._blame_lines(revision_hash, filepath, strategy,
ignore_lines, validated_bugfix_lines)
parent_commit = self._repo.revparse_single('{}^'.format(revision_hash))
blame = self._repo.blame(filepath,
flags=GIT_BLAME_TRACK_COPIES_SAME_FILE,
newest_commit=parent_commit.hex)
for lineno, line in changed_lines:
# returns blamehunk for specific line
try:
bh = blame.for_line(lineno)
except IndexError as e:
# this happens when we have the wrong parent node
bla = 'tried to get file: {}, line: {}, revision: {}, blame commit: {}'.format(
filepath, lineno, revision_hash, str(bh.orig_commit_id))
self._log.error(bla)
raise # this is critical
inducing_commit = self._repo.revparse_single(str(
bh.orig_commit_id))
# start = bh.orig_start_line_number
# lines = bh.lines_in_hunk
# final_start = bh.final_start_line_number
# print(revision_hash, '->', inducing_commit.hex)
# print('original: {}: {}'.format(lineno, line))
# print('{},{}: {},{}'.format(start, lines, final_start, lines))
# blame_lines = []
# for hunk in self._hunks[inducing_commit.hex]:
# if hunk['new_file'] != bh.orig_path:
# continue
# ls = final_start
# for i, blame_line in enumerate(hunk['content'].split('\n')):
# if blame_line[1:].strip() and line[1:].strip() and blame_line[1:] == line[1:]:
# print('blame: {}:{}'.format(ls, blame_line))
# ls += 1
commits.append((inducing_commit.hex, bh.orig_path))
# make unique
return list(set(commits))
def commit_information(self, revision_hash):
obj = self._repo.get(revision_hash)
return {
'author_name':
obj.author.name,
'author_email':
obj.author.email,
'committer_name':
obj.committer.name,
'committer_email':
obj.committer.email,
'committer_date_utc':
datetime.fromtimestamp(obj.commit_time, tz=timezone.utc),
'committer_date':
obj.commit_time,
'committer_date_offset':
obj.commit_time_offset,
'message':
obj.message,
'file_actions':
self._file_actions[revision_hash]
}
def file_actions(self, revision_hash):
return self._file_actions[revision_hash]
def all_files(self, revision_hash):
# 1. checkout repo
self._checkout_revision(revision_hash)
# 2. list files
return self._list_files()
def first_occurence(self, filename):
# file rename tracking is not possible currently in libgit, see:
# https://github.com/libgit2/libgit2/issues/3041
# find first occurence of file with git cli
# git log --follow --diff-filter=A --find-renames=40% foo.js
path = self._path.replace('.git', '')
c = subprocess.run([
'git', 'log', '--all', '--pretty=tformat:"%H %ci"', '--follow',
'--diff-filter=A', '--find-renames=80%', '--', filename
],
cwd=path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if c.returncode != 0:
err = 'Error finding first occurrence of file: {}'.format(filename)
self._log.error(err)
self._log.error(c.stderr)
raise Exception(err)
full = c.stdout.decode('utf-8')
try:
first_line = full.split('\n')[-2]
except IndexError:
if not full:
print('no git log for file {}'.format(filename))
print(full)
raise
first_date = ' '.join(first_line.split(' ')[1:]).replace('"', '')
dt = datetime.strptime(
first_date, '%Y-%m-%d %H:%M:%S %z'
) # we can do this here because we control the input format, %z does not cover +01:00 just +100 (at least in 3.6)
return dt
def tags(self):
regex = re.compile('^refs/tags')
ret = []
for tagref in filter(lambda r: regex.match(r),
self._repo.listall_references()):
tag = self._repo.lookup_reference(tagref)
target = self._repo.lookup_reference(tagref).peel()
ret.append({
'name': tag.name.replace('refs/tags/', ''),
'revision_hash': target.id
})
return ret
def _checkout_revision(self, revision):
"""Checkout via shell, we ignore stdout output."""
path = self._path.replace('.git', '')
c = subprocess.run(['git', 'checkout', '-q', '-f', revision],
cwd=path,
stdout=subprocess.PIPE)
return c.returncode == 0
def _list_files(self):
"""The slower list_files"""
path = self._path.replace('.git', '')
ret = []
for root, dirs, files in os.walk(path):
for file in files:
filepath = os.path.join(root, file)
relative_filepath = filepath.replace(path, '')
ret.append(relative_filepath)
return ret
def _list_files2(self):
"""The faster list_files (relies on find command)"""
path = self._path.replace('.git', '')
lines = subprocess.check_output(['find', '.', '-iname', '*.java'],
cwd=path)
files = []
for f in lines.decode('utf-8').split('\n'):
if f.lower().endswith('.java'):
files.append(f.replace('./', ''))
return files
def _get_hunks(self, commit):
diffs = []
hunks = []
# for initial commit (or orphan commits) pygit2 needs some special attention
initial = False
if not commit.parents:
initial = True
diffs.append((None,
commit.tree.diff_to_tree(context_lines=0,
interhunk_lines=1)))
# we may have multiple parents (merge commit)
for parent in commit.parents:
# we need all information from each parent because in a merge each parent may add different files
tmp = self._repo.diff(parent,
commit,
context_lines=0,
interhunk_lines=1)
tmp.find_similar(self._dopts, self._SIMILARITY_THRESHOLD,
self._SIMILARITY_THRESHOLD)
diffs.append((parent.hex, tmp))
for parent, diff in diffs:
checked_paths = set()
for patch in diff:
if patch.delta.new_file.path in checked_paths:
self._log.warn('already have {} in checked_paths'.format(
patch.delta.new_file.path))
continue
mode = 'X'
if patch.delta.status == 1:
mode = 'A'
elif patch.delta.status == 2:
mode = 'D'
elif patch.delta.status == 3:
mode = 'M'
elif patch.delta.status == 4:
mode = 'R'
elif patch.delta.status == 5:
mode = 'C'
elif patch.delta.status == 6:
mode = 'I'
elif patch.delta.status == 7:
mode = 'U'
elif patch.delta.status == 8:
mode = 'T'
# diff to tree gives D for inital commit otherwise
if initial:
mode = 'A'
# we may have hunks to add
if patch.hunks and commit.hex not in self._hunks.keys():
self._hunks[commit.hex] = []
# add hunks
for hunk in patch.hunks:
# initial is special case
if initial:
content = ''.join(
['+' + l.content for l in hunk.lines])
hunks.append({
'header': hunk.header,
'new_file': patch.delta.new_file.path,
'new_start': hunk.old_start,
'new_lines': hunk.old_lines,
'old_start': hunk.new_start,
'old_lines': hunk.new_lines,
'content': content
})
else:
content = ''.join(
[l.origin + l.content for l in hunk.lines])
hunks.append({
'header': hunk.header,
'new_file': patch.delta.new_file.path,
'new_start': hunk.new_start,
'new_lines': hunk.new_lines,
'old_start': hunk.old_start,
'old_lines': hunk.old_lines,
'content': content
})
return hunks
def _changed_files(self, commit):
changed_files = []
diffs = []
# for initial commit (or orphan commits) pygit2 needs some special attention
initial = False
if not commit.parents:
initial = True
diffs.append((None,
commit.tree.diff_to_tree(context_lines=0,
interhunk_lines=1)))
# we may have multiple parents (merge commit)
for parent in commit.parents:
# we need all information from each parent because in a merge each parent may add different files
tmp = self._repo.diff(parent,
commit,
context_lines=0,
interhunk_lines=1)
tmp.find_similar(self._dopts, self._SIMILARITY_THRESHOLD,
self._SIMILARITY_THRESHOLD)
diffs.append((parent.hex, tmp))
for parent, diff in diffs:
checked_paths = set()
for patch in diff:
if patch.delta.new_file.path in checked_paths:
self._log.warn('already have {} in checked_paths'.format(
patch.delta.new_file.path))
continue
mode = 'X'
if patch.delta.status == 1:
mode = 'A'
elif patch.delta.status == 2:
mode = 'D'
elif patch.delta.status == 3:
mode = 'M'
elif patch.delta.status == 4:
mode = 'R'
elif patch.delta.status == 5:
mode = 'C'
elif patch.delta.status == 6:
mode = 'I'
elif patch.delta.status == 7:
mode = 'U'
elif patch.delta.status == 8:
mode = 'T'
# diff to tree gives D for inital commit otherwise
if initial:
mode = 'A'
# we may have hunks to add
if patch.hunks and commit.hex not in self._hunks.keys():
self._hunks[commit.hex] = []
# add hunks
for hunk in patch.hunks:
# initial is special case
if initial:
content = ''.join(
['+' + l.content for l in hunk.lines])
self._hunks[commit.hex].append({
'header':
hunk.header,
'new_file':
patch.delta.new_file.path,
'new_start':
hunk.old_start,
'new_lines':
hunk.old_lines,
'old_start':
hunk.new_start,
'old_lines':
hunk.new_lines,
'content':
content
})
else:
content = ''.join(
[l.origin + l.content for l in hunk.lines])
self._hunks[commit.hex].append({
'header':
hunk.header,
'new_file':
patch.delta.new_file.path,
'new_start':
hunk.new_start,
'new_lines':
hunk.new_lines,
'old_start':
hunk.old_start,
'old_lines':
hunk.old_lines,
'content':
content
})
# collect line stats
if initial:
fa = {
'lines_added': patch.line_stats[2],
'lines_deleted': patch.line_stats[1],
'changeset_size': len(diff),
'parent': None
}
else:
fa = {
'lines_added': patch.line_stats[1],
'lines_deleted': patch.line_stats[2],
'changeset_size': len(diff),
'parent': parent
}
#if mode == 'R':
# print('R {} -> {}, sim: {}'.format(patch.delta.old_file.path, patch.delta.new_file.path, patch.delta.similarity))
if mode in ['C', 'R']:
changed_file = [
mode, patch.delta.new_file.path,
patch.delta.old_file.path, fa
]
else:
changed_file = [mode, patch.delta.new_file.path, None, fa]
checked_paths.add(patch.delta.new_file.path)
changed_files.append(changed_file)
return changed_files
def collect(self):
# list all branches
for branch in list(self._repo.branches):
self._collect_branch(branch)
# list all tags
for obj in self._repo:
tag = self._repo[obj]
if tag.type == GIT_OBJ_TAG:
self._collect_branch(tag, is_tag=True)
return self._graph
def _collect_branch(self, branch, is_tag=False):
if type(branch) == str:
branch = self._repo.branches[branch]
# add nodes to graph
try:
for c in self._repo.walk(branch.target):
self._graph.add_node(c.hex)
# branch stuff, used for traversing backwards for tags in svn->git conversions
# if c.hex not in self._branches.keys():
# self._branches[c.hex] = []
# what about tags which are also on branches?
# if is_tag:
# self._tags[c.hex] = branch.name
# else:
# self._branches[c.hex].append(branch.branch_name)
# add msg
# self._msgs[c.hex] = c.message
# add days, we use this later for lookup
# day = str(datetime.fromtimestamp(c.commit_time, tz=timezone.utc).date())
# if day not in self._days.keys():
# self._days[day] = []
# self._days[day].append(c.hex)
# add for convenience for OntdekBaanBfs
# self._cdays[c.hex] = day
# add changed files per node
# if c.hex not in self._file_actions.keys():
# self._file_actions[c.hex] = self._changed_files(c)
# still too expensive
# self._create_hunks(c)
# add edges to graph
for c in self._repo.walk(branch.target):
for p in c.parents:
self._graph.add_edge(p.hex, c.hex)
except ValueError as e:
pass
def get(self, request, *args, **kw):
# Process any get params that you may need
# If you don't need to process get params,
# you can skip this part
repo = self.kwargs['resource_id']
try:
specific_repo = repo_model.objects.get(id=repo)
except:
return Response({'detail': 'Repo not found'},
status=status.HTTP_404_NOT_FOUND)
this_repo = Repository(specific_repo.get_repo_path())
directory = ""
if 'directories' in self.kwargs:
directory = self.kwargs['directories']
dir_path = path.join(specific_repo.get_repo_path(), directory)
try:
os.chdir(dir_path)
except:
# returning Response() and raise NotFound is the same
# return Response(status=status.HTTP_404_NOT_FOUND)
raise NotFound(detail="No such dir or file", code=404)
index_tree = this_repo.index
tuplet = []
time2 = None
user = request.user
is_owner = False
is_editor = False
if specific_repo.owner == user:
is_owner = True
if user.is_superuser:
is_owner = True
for editor in specific_repo.editors.all():
if editor.id == user.id:
is_editor = True
empty = False
if this_repo.is_empty:
empty = True
try:
commit = this_repo.revparse_single('HEAD')
tree = commit.tree
folders = []
if directory != "":
item = tree.__getitem__(str(directory))
index_tree.read_tree(item.id)
for entry in index_tree:
name = entry.path
filemode = index_tree[entry.path].mode
type = ""
if filemode is '33188':
type = "tree"
if name in folders:
continue
folders.append(name)
else:
type = "blob"
if "/" in entry.path:
name = entry.path.split("/")[0]
filemode = '100644'
type = "tree"
if name in folders:
continue
folders.append(name)
tuplet.append({
'name': name,
'id': entry.hex,
'type': type,
'filemode': filemode
})
else:
for entry in tree:
tuplet.append({
'name': entry.name,
'id': entry.id.hex,
'type': entry.type,
'filemode': entry.filemode
})
date_handler = lambda obj: ( # noqa: E731
obj.isoformat()
if isinstance(obj,
(datetime.datetime, datetime.date)) else None)
time2 = json.dumps(datetime.datetime.fromtimestamp(
commit.commit_time),
default=date_handler)
dir_hier = directory
main_list = {
'files': tuplet,
'hex': commit.hex,
'message': commit.message,
'author': commit.author.name,
'committer': commit.committer.name,
'time': time2,
'branches': list(this_repo.branches),
'is_owner': is_owner,
'is_empty': empty,
'dir_hier': dir_hier,
'is_editor': is_editor
}
except:
# no files, no initial commit so no head hex
main_list = {
'files': tuplet,
'hex': None,
'message': None,
'author': None,
'committer': None,
'time': None,
'branches': [],
'is_owner': is_owner,
'is_empty': empty,
'is_editor': is_editor,
'dir_hier': None
}
return Response(main_list, status=status.HTTP_200_OK)
class GitMixin(object):
tag_or_remote_regex = re.compile('^refs/(tags|remotes)/(.*)')
def __init__(self):
where = GitOperations.get_repository_location(self.user, self.name)
self.ondisk = Repository(where)
def refresh(self):
creds = GitOperations.get_credentials(self.git_user, self.user)
for remote in self.ondisk.remotes:
remote.credentials = creds
remote.fetch()
# update current reference
master_ref = self.ondisk.lookup_reference('refs/heads/master')
remote_ref = self.ondisk.lookup_reference('refs/remotes/origin/master')
master_ref.set_target(remote_ref.target)
def filter_references(self, regex):
return [ref for ref in self.ondisk.listall_references()
if regex.match(ref)]
def get_commit_time(self, name):
ref = self.ondisk.revparse_single(name)
if isinstance(ref, Tag):
return ref.get_object().commit_time
if isinstance(ref, Commit):
return ref.commit_time
raise GitException('invalid reference: commit time could not be found.') # pragma: no cover
def get_latest_refs(self, count=None):
info = self.filter_references(GitMixin.tag_or_remote_regex)
refs = list(zip(info, map(self.get_commit_time, info)))
refs.sort(key=itemgetter(1), reverse=True)
def ref_info(info):
(ref, commit_time) = info
what, name = GitMixin.tag_or_remote_regex.findall(ref)[0]
return (what, name, commit_time)
refs = map(ref_info, refs)
if not count:
return refs
return islice(refs, count)
def filter_commits(self, flags=0):
all_commits = self.ondisk.walk(self.ondisk.head.target, flags)
emails = [ue.email for ue in self.user.emails.all()]
return filter(lambda commit: commit.author.email in emails, all_commits)
def get_commits(self, count=None):
all_commits = self.filter_commits(GIT_SORT_TOPOLOGICAL)
if not count:
return all_commits
return islice(all_commits, count)
def get_commit_count(self):
return len(list(self.filter_commits()))
def get_shorthand_of_branch(self, branch):
commit = self.ondisk.lookup_branch(branch)
if commit:
return commit.shorthand
return '(none)'
def get_sha1_of_branch(self, branch):
commit = self.ondisk.lookup_branch(branch)
if commit:
return str(commit.get_object().id)[:6]
return '(none)'
def get_numstat(self, commit):
diff = None
try:
previous_commit = self.ondisk.revparse_single(str(commit.id) + '^')
diff = self.ondisk.diff(previous_commit, commit)
except KeyError:
# likely we hit the very first commit.
diff = commit.tree.diff_to_tree(swap=True)
additions, deletions = 0, 0
for patch in diff:
additions += patch.additions
deletions += patch.deletions
return (len(diff), additions, deletions)
def get_first_updated(self):
all_commits = self.ondisk.walk(self.ondisk.head.target,
GIT_SORT_TIME | GIT_SORT_REVERSE)
first_commit = next(all_commits)
return first_commit.commit_time
def get_last_updated(self):
all_commits = self.ondisk.walk(self.ondisk.head.target,
GIT_SORT_TIME)
last_commit = next(all_commits)
return last_commit.commit_time
def get_file_count(self):
diff = self.ondisk.head.get_object().tree.diff_to_tree()
return len([patch.old_file_path for patch in diff])
def get_line_count(self):
diff = self.ondisk.head.get_object().tree.diff_to_tree()
return sum([patch.deletions for patch in diff])
def get_author_count(self):
commits = self.filter_commits()
return len(set([commit.author.email for commit in commits]))
def commits_between(self, start, end):
all_commits = self.filter_commits(GIT_SORT_TIME | GIT_SORT_REVERSE)
starting = dropwhile(lambda obj: obj.commit_time < start, all_commits)
return takewhile(lambda obj: obj.commit_time <= end, starting)
@staticmethod
def by_day(obj):
# we want to group our commit times by the day. so convert
# timestamp -> date -> timestamp
new_date = date.fromtimestamp(obj.commit_time)
new_date += timedelta(days=1)
return timegm(new_date.timetuple())
@staticmethod
def group_by(series):
result = groupby(series, GitMixin.by_day)
return [{'date': commit_date,
'value': len(list(commits))}
for commit_date, commits in result]
def histogram(self, start, end):
series = self.commits_between(start, end)
return GitMixin.group_by(series)
def process(repo, history):
# GET A REPO ON DISK
base = Repository(repo)
base.checkout('HEAD')
file_xsmall = 0
file_small = 0
file_medium = 0
file_large = 0
file_xlarge = 0
hunk_xsmall = 0
hunk_small = 0
hunk_medium = 0
hunk_large = 0
hunk_xlarge = 0
line_xsmall = 0
line_small = 0
line_medium = 0
line_large = 0
line_xlarge = 0
i = 0
while i < len(history) - 1:
print '\rDiff#: ' + str(i + 1) + ' of ' + str(len(history)-1),
t0 = base.revparse_single(history[i].hex)
t1 = base.revparse_single(history[i+1].hex)
try:
diff = base.diff(t0,t1)
except ValueError:
print ''
print 'Value Error'
print ''
i += 1
continue
files = [p for p in diff]
if len(files) == 1:
file_xsmall += 1
if len(files) >= 2 and len(files) <= 4:
file_small += 1
if len(files) >= 5 and len(files) <= 7:
file_medium += 1
if len(files) >= 8 and len(files) <= 10:
file_large += 1
if len(files) >= 11:
file_xlarge += 1
hunksInCommit = 0
linesInCommit = 0
for modfile in files:
hunksInCommit += len(modfile.hunks)
for hunk in modfile.hunks:
for line in hunk.lines:
if line[0] == '-' or line[0] == '+':
linesInCommit += 1
if hunksInCommit <= 1:
hunk_xsmall += 1
if hunksInCommit >= 2 and hunksInCommit <= 8:
hunk_small += 1
if hunksInCommit >= 9 and hunksInCommit <= 17:
hunk_medium += 1
if hunksInCommit >= 18 and hunksInCommit <= 26:
hunk_large += 1
if hunksInCommit >= 27:
hunk_xlarge += 1
if linesInCommit <= 5:
line_xsmall += 1
if linesInCommit >= 6 and linesInCommit <= 46:
line_small += 1
if linesInCommit >= 47 and linesInCommit <= 106:
line_medium += 1
if linesInCommit >= 107 and linesInCommit <= 166:
line_large += 1
if linesInCommit >= 167:
line_xlarge += 1
i += 1
print ''
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('-%Y-%m-%d.%H.%M.%S')
name = repo.replace('/.git', '') + st + '.txt'
output = open(name,'w')
output.write('--------- ' + repo + ' ----------' + '\n')
output.write('Number of Lines Modified:' + '\n')
output.write('x-small: ' + str( + line_xsmall) + '\n')
output.write('small: ' + str(line_small) + '\n')
output.write('medium: ' + str(line_medium) + '\n')
output.write('large: ' + str(line_large) + '\n')
output.write('x-large: ' + str(line_xlarge) + '\n')
output.write('Number of Files Modified:' + '\n')
output.write('x-small: ' + str(file_xsmall) + '\n')
output.write('small: ' + str(file_small) + '\n')
output.write('medium: ' + str(file_medium) + '\n')
output.write('large: ' + str(file_large) + '\n')
output.write('x-large: ' + str(file_xlarge) + '\n')
output.write('Number of Hunks Per Commit' + '\n')
output.write('x-small: ' + str(hunk_xsmall) + '\n')
output.write('small: ' + str(hunk_small) + '\n')
output.write('medium: ' + str(hunk_medium) + '\n')
output.write('large: ' + str(hunk_large) + '\n')
output.write('x-large: ' + str(hunk_xlarge) + '\n')
output.close()
