def init(req):
print "args:", req.args, req.args.path
repo_path = os.path.join(req.args.path,'.git')
if not os.path.isdir(repo_path):
raise Abort("no repository found in '%s' (.git not found)!" % (os.path.abspath(req.args.path)))
repo = Repository(req.args.path)
print "repo references:", repo.listall_references()
xxl_path = os.path.join(req.args.path,'.gitxxl')
if os.path.exists(xxl_path):
print "reinitializing existing XXL repo"
else:
print "initialiling XXL repo"
os.mkdir(xxl_path)
#install commit hooks
dst = os.path.join(repo_path,"hooks","pre-commit")
if not os.path.isfile(dst):
src = os.path.join(sys.exec_prefix,"dist","hooks","pre-commit")
print "installing pre-commit hook from:", src
shutil.copyfile(src, dst)
perm = os.stat(dst)
os.chmod(dst,perm.st_mode | stat.S_IXUSR)
dst = os.path.join(repo_path,"hooks","post-commit")
if not os.path.isfile(dst):
src = os.path.join(sys.exec_prefix,"dist","hooks","post-commit")
print "installing post-commit hook from:", src
shutil.copyfile(src, dst)
perm = os.stat(dst)
os.chmod(dst,perm.st_mode | stat.S_IXUSR)
def install_package(self, pkg_name, expression):
package_cache_path = path.join(APP_CACHE_DIR, pkg_name)
# Check if the package is cached
try:
repo = Repository(package_cache_path)
# TODO: check lastest version
# If lastest is cached, advance
# If no, prompt user
except KeyError as e:
repo = clone_repository(APP_REMOTE['url'], package_cache_path)
# Get suitable version for the expression
versions = utils.get_versions_cached(repo)
version = max_satisfy(versions=versions.keys(), expression='')
package_archive_name = '{0}-{1}.tar'.format(pkg_name, version)
package_archive_path = path.join(APP_ARCHIVE_DIR, package_archive_name)
# Create archive file
# If the file already exists, move forward
if path.isfile(package_archive_path):
with tarfile.open(package_archive_path, 'w') as archive:
repo.write_archive(archive=archive, treeish=versions[version].tree_id)
# TODO: use strategy
# Extract archive to package dir
path_to_package = path.join(USER_PACKAGE_FOLDER, 'user', pkg_name)
tar = tarfile.open(package_archive_path)
tar.extractall(path=path_to_package)
tar.close()
def extract_commits(repos_root, output_path):
# Uncomment code to generate a separate file for each commit.
try:
os.makedirs(output_path)
except FileExistsError as ex:
pass
exec_dir = os.getcwd()
for git_repo in get_immediate_subdirectories(repos_root):
os.chdir(git_repo)
repo = Repository(os.path.join(git_repo, git_folder))
root = etree.Element("commits")
repo_name = os.path.basename(os.path.normpath(git_repo))
print("\n> project: " + repo_name + " extraction started")
for commit in repo.walk(repo.head.target):
stats = get_commit_stats(commit.id)
commit_xml = commit_to_xml(commit, stats)
root.append(commit_xml)
# print(".", end=" ")
print("> project: " + repo_name + ", commit " + str(commit.id) + " processed")
output_xml = xml_to_string(root)
os.chdir(exec_dir)
with open(os.path.join(output_path, repo_name + "_" + output_commit_file), "w") as file:
file.write(output_xml)
print("\n> project: " + repo_name + " extraction finished")
def commit_new_tickets_to_git(queue, repo_path):
global last_push_ts
#repo
repo = Repository(os.path.join(repo_path, '.git'))
index = repo.index
author = Signature('yourname', 'youremail')
while True:
#write tickets into file
ticket = queue.get(block=True)
#filename format is yyyy-mm-dd
d = datetime.utcnow().date()
filename = '%s.txt' % d.strftime('%Y-%m-%d')
f = open(os.path.join(repo_path, filename), 'ab')
f.write('%s\n' % ticket.toJson())
f.close()
#commit
index.add(filename)
index.write()
oid = index.write_tree()
repo.create_commit('HEAD', author, author, ticket.toJson(), oid, [repo.head.oid])
#push
d_ts = datetime.utcnow()
if last_push_ts is None or d_ts > (last_push_ts + timedelta(seconds = 60)):
push_to_github(repo_path)
last_push_ts = datetime.utcnow()
class Repo(object):
def __init__(self,path,username,password):
self.repo = Repository(path)
self.username = username
self.password = password
self.path=path
def commit(self,file,refspec,email,author_username,message):
self.repo.index.add(file)
index = self.repo.index
index.write()
tree = self.repo.index.write_tree()
author = Signature(author_username,email)
self.repo.create_commit(refspec,
author,
author,message,tree, [self.repo.head.get_object().hex] )
def commit_fallback(self,message):
return subprocess.check_output(["git","commit","-m",message],cwd=self.path)
def push(self,refspec):
credentials = UserPass(self.username,self.password)
remoteCall = RemoteCallbacks(credentials)
remo = self.repo.remotes["origin"]
remo.push(refspec,remoteCall)
def get_new_articles():
blog = PelicanBlog()
content_dir = blog.get_content_directory()
repo = Repository(os.path.abspath(os.path.dirname(__file__)))
diff = repo.revparse_single("HEAD").tree.diff_to_tree()
existing_articles = set(os.path.relpath(obj.old_file_path, content_dir)
for obj in diff
if obj.old_file_path.startswith(content_dir))
all_articles = set(blog.get_posts())
new_articles = {art for art in all_articles - existing_articles
if blog.get_post_lang(art) in (TWITTER_LANGUAGE, "")}
new_titles = []
repo.index.read()
for newart in new_articles:
title = blog.get_post_title(newart)
yield Article(title, blog.get_post_url(newart),
blog.get_post_authors(newart))
new_titles.append(title)
repo.index.add(os.path.join(content_dir, newart))
blogger = Signature(repo.config["user.name"], repo.config["user.email"])
repo.create_commit("HEAD", blogger, blogger,
"[BLOG] %s" % ", ".join(new_titles),
repo.index.write_tree(), [repo.head.peel().oid])
repo.index.write()
# TODO(v.markovtsev): implement git push using pygit2
subprocess.call(("git", "push", "origin", repo.head.shorthand))
class GitRepositoryTestCase(unittest.TestCase):
def setUp(self):
self.repo_path = mkdtemp()
init_repository(self.repo_path, False)
self.repo = Repository(self.repo_path)
def tearDown(self):
try:
rmtree(self.repo_path)
except:
pass
def test_create_repo(self):
repo_path = mkdtemp()
try:
GitRepository.create_repo(repo_path)
for f in [os.path.join('content', 'attachments', 'mercurial.png'),
os.path.join('content', 'post', 'example-post.rst'),
os.path.join('content', 'post', 'lorem-ipsum.rst'),
os.path.join('content', 'about.rst'),
os.path.join('static', 'screen.css'),
os.path.join('templates', 'base.html'),
os.path.join('templates', 'posts.html'),
os.path.join('templates', 'post_list.html'),
'config.yaml', '.gitignore', '.git']:
self.assertTrue(os.path.exists(os.path.join(repo_path, f)),
'Not found: %s' % f)
finally:
rmtree(repo_path)
def test_get_changectx_rev_default(self):
git_repo = GitRepository(self.repo_path)
with codecs.open(os.path.join(self.repo_path, 'foo.rst'), 'w',
encoding='utf-8') as fp:
fp.write('foo')
sign = Signature('foo', '*****@*****.**')
tree = self.repo.TreeBuilder().write()
self.repo.index.add('foo.rst')
self.repo.create_commit('refs/heads/master', sign, sign, 'foo', tree,
[])
self.assertTrue(isinstance(git_repo.get_changectx(REVISION_DEFAULT),
ChangeCtxDefault),
'changectx object is not an instance of '
'ChangeCtxDefault')
def test_get_changectx_rev_working_dir(self):
git_repo = GitRepository(self.repo_path)
with codecs.open(os.path.join(self.repo_path, 'foo.rst'), 'w',
encoding='utf-8') as fp:
fp.write('foo')
sign = Signature('foo', '*****@*****.**')
tree = self.repo.TreeBuilder().write()
self.repo.index.add('foo.rst')
self.repo.create_commit('refs/heads/master', sign, sign, 'foo', tree,
[])
self.assertTrue(
isinstance(git_repo.get_changectx(REVISION_WORKING_DIR),
ChangeCtxWorkingDir),
'changectx object is not an instance of ChangeCtxWorkingDir')
def get_commit_activity(self, project):
from datetime import date, timedelta
from pygit2 import Repository
from pygit2 import GIT_SORT_TIME
repo = Repository(project.gitrepo)
weeks = self.get_weeks()
for commit in repo.walk(repo.head.oid, GIT_SORT_TIME):
commit_time = date.fromtimestamp(commit.commit_time)
commit_week = commit_time - timedelta(days=commit_time.weekday())
if commit_week not in weeks:
continue
weeks[commit_week]['mine'] += 1
counts = []
max = 0
for k in sorted(weeks.iterkeys()):
counts.append({
"week": k.isoformat(),
"mine": weeks[k]['mine'],
"others": weeks[k]['others'],
})
return counts
def get_bug_commit_ratio_per_file(git_folder = ".git/", output_file):
result = []
exec_dir = os.getcwd()
repo = Repository(os.path.join(git_repo, git_folder))
os.chdir(git_repo)
for commit in repo.walk(repo.head.target):
touched_files = get_touched_files(commit)
for file in touched_files:
file_data = [f for f in result if f['file_name'] == file]
if file_data:
file_data = file_data[0]
file_data['commit_num'] += 1
if bug_related:
file_data['bug_commit_num'] += 1
else:
result.append({'file_name': file,
'commit_num': 1,
'bug_commit_num': 1 if bug_related else 0})
os.chdir(exec_dir)
for entry in result:
entry['bug_commit_ratio'] = entry['bug_commit_num'] / entry['commit_num']
with open(output_file, "w", newline='') as output:
writer = csv.DictWriter(output, csv_header)
writer.writeheader()
writer.writerows(result)
def getHist(repo):
base = Repository(repo)
base.checkout('HEAD')
history = []
for commit in base.walk(base.head.target, GIT_SORT_TOPOLOGICAL | GIT_SORT_REVERSE):
history.append(commit)
return history
def get_blame_test():
git_repo = "/Users/aniketp/Workspace/Capstone/Source/eclipse/eclipse.platform.runtime"
sample_file = "bundles/org.eclipse.core.jobs/src/org/eclipse/core/internal/jobs/JobManager.java"
exec_dir = os.getcwd()
repo = Repository(os.path.join(git_repo, git_folder))
os.chdir(git_repo)
blameObj = repo.blame(sample_file, min_line=1, max_line=20)
def get_and_update_repo_cache(repo_path):
cache_filename = '%s-stats.cache' % repo_path
if os.path.exists(cache_filename):
with open(cache_filename) as f:
data = load(f)
else:
data = {
'author_to_month_to_additions': defaultdict(defaultdict_int),
'author_to_month_to_deletions': defaultdict(defaultdict_int),
'author_to_month_to_commits': defaultdict(defaultdict_int),
'day_to_count': defaultdict(defaultdict_int),
'latest_sha': None,
}
repo = Repository(repo_path)
count = 0
for commit in repo.walk(repo.head.target, GIT_SORT_TOPOLOGICAL):
count += 1
if commit.type == GIT_OBJ_COMMIT:
if data['latest_sha'] == commit.hex:
break
if not commit.message.lower().startswith('merge'):
try:
d = repo.diff('%s^' % commit.hex, commit)
except KeyError:
# First commit!
break
patches = list(d)
additions = sum([p.additions for p in patches])
deletions = sum([p.deletions for p in patches])
author = author_aliases.get(commit.author.email, commit.author.email)
day = date.fromtimestamp(commit.commit_time)
data['day_to_count']['Lines'][day] += additions
data['day_to_count']['Lines'][day] -= deletions
if additions > 1000 and deletions < 5 and commit.hex not in whitelist_commits:
if commit.hex not in blacklist_commits:
print 'WARNING: ignored %s looks like an embedding of a lib (message: %s)' % (commit.hex, commit.message)
continue
if (additions > 3000 or deletions > 3000) and commit.hex not in whitelist_commits:
if commit.hex not in blacklist_commits and additions != deletions: # Guess that if additions == deletions it's a big rename of files
print 'WARNING: ignored %s because it is bigger than 3k lines. Put this commit in the whitelist or the blacklist (message: %s)' % (commit.hex, commit.message)
continue
month = date(day.year, day.month, 1)
data['author_to_month_to_additions'][author][month] += additions
data['author_to_month_to_deletions'][author][month] += deletions
data['author_to_month_to_commits'][author][month] += 1
if data['latest_sha'] is None:
data['latest_sha'] = commit.hex
with open(cache_filename, 'w') as f:
dump(data, f)
return data
def gets(cls, path, max_count=100, order=GIT_SORT_TIME):
"""gets commits from a git repository.
:param path: The normalized path to the git repository.
:param max_count: max count of commits.
:param order: order commits list."""
repo = Repository(path)
return [cls(c.hex, [p.hex for p in c.parents])
for c in islice(repo.walk(repo.head.target, order),
max_count)]
def sync_handler(fork_from: str, from_sha: str, repo_name: str,
ticket_id: int, pr_url: str):
output_path = '{}.txt'.format(pr_url.split('/', 3)[3].rsplit('/', 2)[0])
output_path = os.path.join(WORK_DIR, output_path.replace('/', '_'))
work_tree = os.path.join(WORK_DIR, fork_from)
parent_path = os.path.dirname(work_tree)
if not os.path.exists(parent_path):
os.makedirs(parent_path)
if not os.path.exists(work_tree):
repo = clone_repository(
'{0}{1}.git'.format(GITHUB_URL, fork_from), work_tree)
else:
repo = Repository(work_tree)
remote_name = repo_name.split('/')[0]
update_remote(work_tree, repo, repo_name, remote_name)
if remote_name == 'origin':
commit = repo.revparse_single(from_sha)
repo.checkout_tree(commit, strategy=GIT_CHECKOUT_FORCE)
else:
ref_name = 'refs/pull/{0}/head'.format(ticket_id)
try:
repo.create_reference(ref_name, from_sha)
except ValueError:
pass
ref = repo.lookup_reference(ref_name)
repo.checkout(ref, strategy=GIT_CHECKOUT_FORCE)
cwd = os.getcwd()
os.chdir(work_tree)
subprocess.call(
'{} . --output-file={}'.format(FLAKE8_EXECUTABLE, output_path),
shell=True)
os.chdir(cwd)
return output_path
def commitInfo(repo, history):
# GET A REPO ON DISK
base = Repository(repo)
base.checkout('HEAD')
# MOVE THROUGH THE SYSTEM HISTORY FROM NEWEST TO OLDEST COMMIT
for commit in history:
print 'Date/Time: ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(commit.commit_time))
print 'Comment Hex: ' + commit.hex
print 'Message: ' + commit.message.rstrip('\n')
print ''
def __init__(self):
pubsub = PubSub()
events_dispatcher = EventsDispatcher(pubsub)
display = self.display_surface()
repo = Repository(config["repo_path"])
branches = repo.listall_branches()
drawer = Drawer(repo, branches, display, pubsub)
pubsub.sub("on_program_exit", self.exit_program)
self.main_loop = MainLoop(display, drawer, events_dispatcher)
self.init_screen()
def get_deleted_files(path_to_repository):
"""Utility method for getting the deleted files from a Git repository.
Args:
path_to_repository (str): Path to the Git repository
Returns:
List(str): List of filenames of all deleted files in the provided repository.
"""
repo = Repository(path_to_repository)
status_entries = repo.status()
return [path for path, st in status_entries.items() if st & _STATI_DELETED]
def get_changed_files(path_to_repository):
"""Utility method for getting the currently changed files from a Git repository.
Args:
path_to_repository (str): Path to the Git repository
Returns:
List(str): List of filenames of all changed files in the provided repository.
"""
repo = Repository(path_to_repository)
status_entries = repo.status()
return [path for path, st in status_entries.items() if st & _STATI_CONSIDERED_FOR_PRECOMMIT]
def get_current_timestamp(path_to_repository):
"""Utility method for getting the timestamp of the last commit from a Git repository.
Args:
path_to_repository (str): Path to the Git repository
Returns:
str: The timestamp of the last commit in the provided repository.
"""
repo = Repository(path_to_repository)
head = repo.revparse_single('HEAD')
return head.commit_time
def get_current_branch(path_to_repository):
"""Utility method for getting the current branch from a Git repository.
Args:
path_to_repository (str): Path to the Git repository
Returns:
str: The current branch in the provided repository.
"""
repo = Repository(path_to_repository)
head = repo.lookup_reference("HEAD").resolve()
return head.shorthand
def get_churn_per_commit(git_repo, output_file):
touched_files = []
exec_dir = os.getcwd()
repo = Repository(os.path.join(git_repo, git_folder))
os.chdir(git_repo)
for commit in repo.walk(repo.head.target):
touched_files = get_touched_files(commit)
with open(output_file, "w", newline='') as output:
writer = csv.DictWriter(output, csv_header)
writer.writeheader()
writer.writerows(touched_files)
def shift(amount, repo_path):
repo = Repository(repo_path)
head = repo.lookup_reference('HEAD').resolve()
adder = partial(add, amount=amount)
changelog = dict()
reference = REF_FMT.format(time=time(), pid=getpid())
for commit in repo.walk(head.oid, GIT_SORT_REVERSE | GIT_SORT_TOPOLOGICAL):
newmsg, nsubs = ISSUE_RE.subn(adder, commit.message)
if nsubs != 0 or any(pnt.oid in changelog for pnt in commit.parents):
parents = [changelog.get(c.oid, c.oid) for c in commit.parents]
new_oid = repo.create_commit(reference, commit.author,
commit.committer, newmsg, commit.tree.oid, parents)
changelog[commit.oid] = new_oid
return changelog, reference
class Git(object):
r"""
Interact with a git repository.
"""
def __init__(self, gitdir):
r"""
Take a path to the git repository. Other methods interact with
this git repository.
"""
self.repo = Repository(gitdir)
def branches(self):
r"""
Return the list of a branch name and its last commit id.
"""
return self._refs('heads')
def tags(self):
r"""
Return the list of a tag name and its last commit id.
"""
return self._refs('tags')
def _refs(self, type):
refs = {}
pattern = re.compile(r'refs/%s/(.*)$' % type)
for ref in self.repo.listall_references():
m = pattern.match(ref)
if m:
reference = self.repo.lookup_reference(ref)
refs[m.group(1)] = reference.hex
return refs
def create_branch(self, name, target):
r"""
Create new branch.
"""
if not is_valid_value(name):
raise InvalidParamException("name is required")
if not is_valid_hex(target):
raise InvalidParamException("target is required")
target = sha_hex2bin(target)
try:
self.repo.create_reference('refs/heads/%s' % name, target)
except Exception, e:
raise InvalidParamException(str(e))
return True
def update_repo(reponame):
""" For a given path to a repo, pull/rebase the last changes if
it can, add/remove/commit the new changes and push them to the
remote repo if any.
:kwarg reponame, full path to a git repo.
"""
LOG.info('Processing %s' % reponame)
if not os.path.exists(reponame):
raise GitSyncError(
'The indicated working directory does not exists: %s' %
reponame)
try:
repo = Repository(reponame)
except Exception as err:
print(err)
raise GitSyncError(
'The indicated working directory is not a valid git '
'repository: %s' % reponame)
index = repo.index
dopush = False
origin = None
index = repo.index
## Add or remove to staging the files according to their status
if repo.status:
status = repo.status()
for filepath, flag in status.items():
if flag == GIT_STATUS_WT_DELETED:
msg = 'Remove file %s' % filepath
LOG.info(msg)
index.remove(filepath)
docommit(repo, index, msg)
dopush = True
elif flag == GIT_STATUS_WT_NEW:
msg = 'Add file %s' % filepath
LOG.info(msg)
index.add(filepath)
docommit(repo, index, msg)
dopush = True
elif flag == GIT_STATUS_WT_MODIFIED:
msg = 'Change file %s' % filepath
LOG.info(msg)
index.add(filepath)
docommit(repo, index, msg)
dopush = True
return dopush
def get_commit_churn(repo_path):
exec_dir = os.getcwd()
repo = Repository(os.path.join(repo_path, git_folder))
os.chdir(repo_path)
debt_commits = get_debt_commits(repo_path) #commit, commit_ts, fiel_path
file_churn = []
for commit in repo.walk(repo.head.target):
print (commit.id)
curr_commit_ts = int(str(get_unix_timestamp(str(commit.id))).replace("\n", ""))
debt_commit_flag = [f for f in debt_commits if f['commit'] == str(commit.id)]
#print (str(debt_commit_flag))
is_debt_commit = 0
if debt_commit_flag:
is_debt_commit = 1
touched_files = get_touched_files(commit)
for strFileChurn in touched_files:
added_lines, deleted_lines, file_name = strFileChurn.split("\t")
file_commit_flag = [f for f in debt_commits if f['file_path'] == file_name]
is_file_debt = 0
if file_commit_flag:
file_commit_flag = file_commit_flag[0]
debt_commit_commit_ts = int(file_commit_flag['commit_ts'])
if curr_commit_ts >= debt_commit_commit_ts:
is_file_debt = 1
try:
file_churn.append({
'commit':str(commit.id),
'commit_ts':curr_commit_ts,
'file_name':file_name,
'added_lines':added_lines,
'deleted_lines':deleted_lines,
'commit_debt':is_debt_commit,
'file_debt':is_file_debt,
})
except (AttributeError):
continue;
os.chdir(exec_dir)
with open(output_file, "w", newline='') as output:
writer = csv.DictWriter(output, csv_header)
writer.writeheader()
writer.writerows(file_churn)
def main():
contrib_data_dir = sys.argv[1]
git_repo = sys.argv[2]
output_file = sys.argv[3]
result = []
exec_dir = os.getcwd()
for contrib_file in os.listdir(contrib_data_dir):
release = contrib_file[len(contrib_file_prefix):][:-len(contrib_file_extension)]
contrib_file = os.path.join(contrib_data_dir, contrib_file)
top_devs = {}
with open(contrib_file, newline="") as csv_file:
for row in csv.DictReader(csv_file):
top_dev = row['top_single_dev_contribution_knowledge'].split(":")[0]
if top_dev in top_devs:
top_devs[top_dev] += 1
else:
top_devs[top_dev] = 1
os.chdir(git_repo)
call(["git", "checkout", "tags/" + release])
os.chdir(exec_dir)
for top_dev in top_devs:
author_commit_count = 0
commit_count = 0
repo = Repository(os.path.join(git_repo, git_folder))
for commit in repo.walk(repo.head.target):
commit_count += 1
if commit.author.name == top_dev:
author_commit_count += 1
result.append({'release': release,
'release_commits': commit_count,
'top_dev': top_dev,
'files_owned': top_devs[top_dev],
'dev_commits': author_commit_count})
with open(output_file, 'w', newline='') as output:
writer = csv.DictWriter(output, csv_header)
writer.writeheader()
writer.writerows(result)
class RepositoryProcessor(object):
def __init__(self, repository_path):
self.repo = GitRepository(repository_path + '/.git')
self.users = {}
def get_bages_processors_for_user(self, email):
if email in self.users:
return self.users[email]
self.users[email] = []
for badge_class in initialize_badge_classes():
logging.info(u'Initializing badge class [%s] for user [%s]' % (str(badge_class), email))
self.users[email].append(badge_class(email))
return self.users[email]
def process(self):
# returns the json of the collaborators
for commit in [c for c in self.repo.walk(self.repo.head.oid, GIT_SORT_TIME)][::-1]:
for badge in self.get_bages_processors_for_user(commit.author.email):
badge.process_commit(commit, datetime.fromtimestamp(commit.commit_time))
result = []
for user_email, badges in self.users.items():
user = {"email": user_email, "badges": []}
result.append(user)
for badge in badges:
if isinstance(badge, Badge):
if badge.award_this():
user['badges'].append({"badge_slug": badge.slug})
else:
user.update(badge.update_data())
user.update(count_modifications_by_user(user_email, self.repo.path))
print user
return result
class GitObjectRenderer:
def __init__(self, reponame, cache, revspec, path):
self.repo = Repository(reponame)
self.revspec = revspec
self.path = path
self.object_map = {
Blob : self._render_blob,
Commit : self._render_commit,
Tag : self._render_tag
}
self.cache = cache
def _render_blob(self, rev):
data = rev.data.decode('utf-8', errors='ignore')
formatter = HtmlFormatter(linenos='inline')
lexer = guess_lexer(data)
blob = highlight(data, lexer, formatter)
return render_template("objects/blob.html",
repo = self.repo,
revspec = 'master', blob = blob, path = self.path)
def _render_commit(self, rev):
entries = self.repo[rev.tree[self.path].oid] if self.path else rev.tree
cached = []
for entry in entries:
cache_id = "{0}-commit-{1}".format(self.repo.get_name(), entry.hex)
hit = self.cache.get(cache_id)
if hit is not None:
cached.append((entry, hit))
else:
cached.append((entry, None))
files = sorted(cached, key=lambda x: x[0].filemode)
return render_template("objects/tree.html",
repo = self.repo,
revspec = self.revspec, rev = rev, files = files,
path = self.path)
def _render_tag(self, rev):
obj = self.repo[rev.target]
url = url_for('obj', reponame=self.repo.get_name(), revspec = obj.hex)
return redirect(url)
def render_obj(self):
rev = self.repo.revparse_single(self.revspec)
return self.object_map[type(rev)](rev)
def git_is_clean(srcdir, project):
repo = Repository(os.path.join(srcdir, project.workspace_path, ".git"))
for _, b in iteritems(repo.status()):
if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT:
return False, "has uncommitted changes"
if repo.head_is_detached:
return False, "has detached HEAD"
origin = get_origin(repo, project)
if not origin:
return False, "has no upstream remote"
remote_refs = []
local_refs = {}
for refname in repo.listall_references():
if refname.startswith("refs/remotes/%s/" % origin.name):
ref = repo.lookup_reference(refname)
if ref.type == GIT_REF_OID:
remote_refs.append(ref.target)
elif not refname.startswith("refs/remotes/"):
ref = repo.lookup_reference(refname)
if ref.type == GIT_REF_OID:
local_refs[ref.peel().id] = refname
if not remote_refs:
return False, "has no upstream remote branches"
if not local_refs:
return False, "has no local branches"
if not repo.lookup_branch("%s/%s" % (origin.name, project.master_branch), GIT_BRANCH_REMOTE):
return False, "has no upstream master branch"
for remote_ref in remote_refs:
for commit in repo.walk(remote_ref):
if commit.id in local_refs:
del local_refs[commit.id]
if local_refs:
return False, "has local commits: %s" % ", ".join(["'%s'" % name for _, name in iteritems(local_refs)])
return True, ""
def cross_reference_commits_with_bug_reports():
repos_root = sys.argv[1]#"/home/kevin/Desktop/eclipse-platform"#
bug_reports_file = sys.argv[2]#"/home/kevin/Downloads/eclipse-bugs.csv"#
with open(bug_reports_file, newline="") as csv_file:
bug_reports = [{"id": bug["id"],
"creation_time": datetime.strptime(bug["creation_time"], bug_date_format),
"closed_time": datetime.strptime(bug["closed_time"], bug_date_format)}
for bug in csv.DictReader(csv_file)]
os.makedirs(output_root_path)
for git_repo in get_immediate_subdirectories(repos_root):
repo_name = os.path.basename(os.path.normpath(git_repo))
repo = Repository(os.path.join(git_repo, git_folder))
bug_related_commits = [commit for commit in repo.walk(repo.head.target) if is_bug_related(commit)]
root = etree.Element("commits")
count = 0
for bug_report in bug_reports:
# This may actually hurt the detection
bug_related_commits_within_bug_life = \
[c for c in bug_related_commits
if bug_report['creation_time'] <= datetime.fromtimestamp(c.commit_time) <= bug_report['closed_time']]
# for commit in bug_related_commits:
for commit in bug_related_commits_within_bug_life:
if are_related(commit, bug_report):
commit_xml = commit_to_xml(commit)
commit_xml.set("related_bug", bug_report["id"])
root.append(commit_xml)
count += 1
print("repo: " + repo_name + ", bug: " + bug_report["id"] + " processed")
# if count > 10:
# break
root.set("count", str(count))
output_xml = xml_to_string(root)
with open(os.path.join(output_root_path, repo_name + "_" + output_commit_file), "w") as file:
file.write(output_xml)
class ChangeCtxDefault(ChangeCtx):
"""Class with the specific implementation details for the change context
of the default revision state of the repository. It inherits the common
implementation from the class :class:`ChangeCtxBase`.
"""
def __init__(self, repo_path):
self._repo_path = repo_path
self._repo = Repository(self._repo_path)
self._ctx = self._repo[self.revision_id]
@locked_cached_property
def files(self):
def r(_files, repo, tree, prefix=None):
for entry in tree:
obj = repo[entry.oid]
filename = prefix and (prefix + '/' + entry.name) or entry.name
if obj.type == GIT_OBJ_TREE:
r(_files, repo, obj, filename)
elif obj.type == GIT_OBJ_BLOB:
_files.append(filename)
else:
raise RuntimeError('Invalid object: %s' % filename)
f = []
r(f, self._repo, self._ctx.tree)
return sorted(f)
@locked_cached_property
def revision_id(self):
"""This property should be cached because the lookup_reference method
reloads itself.
"""
try:
ref = self._repo.lookup_reference('refs/heads/master')
except Exception:
raise RuntimeError('Branch "master" not found!')
return ref.target
def needs_reload(self):
try:
ref = self._repo.lookup_reference('refs/heads/master')
except Exception:
return True
return self.revision_id != ref.target
def filectx_needs_reload(self, filectx):
try:
ref = self._repo.lookup_reference('refs/heads/master')
except Exception:
raise RuntimeError('Branch "master" not found!')
return filectx._changectx.oid != ref.target
def published(self, date, now):
return date <= now
def etag(self, filectx):
return 'blohg-%i-%i-%s' % (filectx.mdate or filectx.date,
len(filectx.data),
adler32(filectx.path.encode('utf-8'))
& 0xffffffff)
def get_filectx(self, path):
return FileCtx(self._repo, self._ctx, path)
# Read the configuration file with parameters,
# location of configuration file - as in production system
asa_parameters = configread(conf_file, 'CiscoASA', 'enablepw')
git_parameters = configread(conf_file, 'GIT', 'author_name', 'committer_name',
'author_email', 'committer_email')
nd_parameters = configread(conf_file, 'NetworkDevice', 'login', 'password',
'slack_hook', 'device_file', 'savedir')
if not os.path.isdir(nd_parameters['savedir']):
os.makedirs(nd_parameters['savedir'])
# Try to create object repository if repository is already exist
try:
repo = Repository(nd_parameters['savedir'])
except KeyError:
repo = init_repository(nd_parameters['savedir'])
git_init = True
# Build the index of repository
index = repo.index
device_list_file = open(nd_parameters['device_file'])
# Parse the device file
for device_line in device_list_file:
device_params = device_line.split(':')
device_type = device_params[0]
device_name = device_params[1]
device_ip = device_params[2]
class GitBareBackend(object):
nb_transactions = 0
def __init__(self, path):
self.path = abspath(path) + '/'
# Open database
self.path_data = '%s/database/' % self.path
if not lfs.is_folder(self.path_data):
error = '"%s" should be a folder, but it is not' % path
raise ValueError, error
# Open repository
self.repo = Repository(self.path_data)
# Read index
try:
tree = self.repo.head.peel(GIT_OBJ_TREE)
self.repo.index.read_tree(tree.id)
except:
pass
# Check git commiter
try:
_, _ = self.username, self.useremail
except:
print '========================================='
print 'ERROR: Please configure GIT commiter via'
print ' $ git config --global user.name'
print ' $ git config --global user.email'
print '========================================='
raise
@classmethod
def init_backend(cls, path, init=False, soft=False):
init_repository('{0}/database'.format(path), bare=True)
#######################################################################
# Internal utility functions
#######################################################################
def _call(self, command):
"""Interface to cal git.git for functions not yet implemented using
libgit2.
"""
popen = Popen(command, stdout=PIPE, stderr=PIPE, cwd=self.path_data)
stdoutdata, stderrdata = popen.communicate()
if popen.returncode != 0:
raise EnvironmentError, (popen.returncode, stderrdata)
return stdoutdata
@lazy
def username(self):
cmd = ['git', 'config', '--get', 'user.name']
try:
username = self._call(cmd).rstrip()
except EnvironmentError:
raise ValueError(
"Please configure 'git config --global user.name'")
return username
@lazy
def useremail(self):
cmd = ['git', 'config', '--get', 'user.email']
try:
useremail = self._call(cmd).rstrip()
except EnvironmentError:
raise ValueError(
"Please configure 'git config --global user.email'")
return useremail
def _resolve_reference(self, reference):
"""This method returns the SHA the given reference points to. For now
only HEAD is supported.
FIXME This is quick & dirty. TODO Implement references in pygit2 and
use them here.
"""
# Case 1: SHA
if len(reference) == 40:
return reference
# Case 2: reference
reference = self.repo.lookup_reference(reference)
try:
reference = reference.resolve()
except KeyError:
return None
return reference.target
def normalize_key(self, path, __root=None):
# Performance is critical so assume the path is already relative to
# the repository.
key = __root.resolve(path)
if key and key[0] == '.git':
err = "bad '{0}' path, access to the '.git' folder is denied"
raise ValueError(err.format(path))
return '/'.join(key)
def handler_exists(self, key):
tree = self.repo.head.peel(GIT_OBJ_TREE)
try:
tree[key]
except:
return False
return True
def get_handler_names(self, key):
try:
tree = self.repo.head.peel(GIT_OBJ_TREE)
if key:
tree_entry = tree[key]
if tree_entry.type == 'blob':
raise ValueError
tree = self.repo[tree_entry.id]
except:
yield None
else:
for item in tree:
yield item.name
def get_handler_data(self, key):
tree = self.repo.head.peel(GIT_OBJ_TREE)
tree_entry = tree[key]
blob = self.repo[tree_entry.id]
return blob.data
def get_handler_mimetype(self, key):
data = self.get_handler_data(key)
return magic_from_buffer(data)
def handler_is_file(self, key):
return not self.handler_is_folder(key)
def handler_is_folder(self, key):
repository = self.repo
if key == '':
return True
else:
tree = repository.head.peel(GIT_OBJ_TREE)
tree_entry = tree[key]
return tree_entry.type == 'tree'
def get_handler_mtime(self, key):
# FIXME
return datetime.utcnow().replace(tzinfo=fixed_offset(0))
def traverse_resources(self):
tree = self.repo.head.peel(GIT_OBJ_TREE)
yield self.get_resource('/')
for name in self.get_names(tree):
if name[-9:] == '.metadata' and name != '.metadata':
yield self.get_resource('/' + name[:-9])
def get_names(self, tree, path=''):
for entry in tree:
base_path = '{0}/{1}'.format(path, entry.name)
yield base_path
if entry.filemode == GIT_FILEMODE_TREE:
sub_tree = self.repo.get(entry.hex)
for x in self.get_names(sub_tree, base_path):
yield x
def do_transaction(self, commit_message, data, added, changed, removed,
handlers):
self.nb_transactions += 1
# Get informations
git_author, git_date, git_msg, docs_to_index, docs_to_unindex = data
git_msg = commit_message or git_msg or 'no comment'
# List of Changed
added_and_changed = list(added) + list(changed)
# Build the tree from index
index = self.repo.index
for key in added_and_changed:
handler = handlers.get(key)
blob_id = self.repo.create_blob(handler.to_str())
entry = IndexEntry(key, blob_id, GIT_FILEMODE_BLOB_EXECUTABLE)
index.add(entry)
for key in removed:
index.remove(key)
git_tree = index.write_tree()
# Commit
self.git_commit(git_msg, git_author, git_date, tree=git_tree)
def git_commit(self, message, author=None, date=None, tree=None):
"""Equivalent to 'git commit', we must give the message and we can
also give the author and date.
"""
# Tree
if tree is None:
#tree = self.index.write_tree()
raise ValueError('Please give me a tree')
# Parent
parent = self._resolve_reference('HEAD')
parents = [parent] if parent else []
# Committer
when_time = time.time()
when_offset = -(time.altzone if time.daylight else time.timezone)
when_offset = when_offset / 60
name = self.username
email = self.useremail
committer = Signature(name, email, when_time, when_offset)
# Author
if author is None:
author = (name, email)
if date:
if date.tzinfo:
from pytz import utc
when_time = date.astimezone(utc) # To UTC
when_time = when_time.timetuple() # As struct_time
when_time = timegm(when_time) # To unix time
when_offset = date.utcoffset().seconds / 60
else:
err = "Worktree.git_commit doesn't support naive datatime yet"
raise NotImplementedError, err
author = Signature(author[0], author[1], when_time, when_offset)
# Create the commit
return self.repo.create_commit('HEAD', author, committer, message,
tree, parents)
def abort_transaction(self):
# TODO: Remove created blobs
pass
class GitRepo:
"""A class that manages a git repository.
This class enables versiong via git for a repository.
You can stage and commit files and checkout different commits of the repository.
"""
path = ''
pathspec = []
repo = None
callback = None
author_name = 'QuitStore'
author_email = '*****@*****.**'
gcProcess = None
def __init__(self, path, origin=None, gc=False):
"""Initialize a new repository from an existing directory.
Args:
path: A string containing the path to the repository.
origin: The remote URL where to clone and fetch from and push to
"""
logger = logging.getLogger('quit.core.GitRepo')
logger.debug('GitRepo, init, Create an instance of GitStore')
self.path = path
self.gc = gc
if not exists(path):
try:
makedirs(path)
except OSError as e:
raise Exception('Can\'t create path in filesystem:', path, e)
try:
self.repo = Repository(path)
except KeyError:
pass
except AttributeError:
pass
if origin:
self.callback = QuitRemoteCallbacks()
if self.repo:
if self.repo.is_bare:
raise QuitGitRepoError('Bare repositories not supported, yet')
if origin:
# set remote
self.addRemote('origin', origin)
else:
if origin:
# clone
self.repo = self.cloneRepository(origin, path, self.callback)
else:
self.repo = init_repository(path=path, bare=False)
def cloneRepository(self, origin, path, callback):
try:
repo = clone_repository(url=origin,
path=path,
bare=False,
callbacks=callback)
return repo
except Exception as e:
raise QuitGitRepoError(
"Could not clone from: {} origin. {}".format(origin, e))
def addall(self):
"""Add all (newly created|changed) files to index."""
self.repo.index.read()
self.repo.index.add_all(self.pathspec)
self.repo.index.write()
def addfile(self, filename):
"""Add a file to the index.
Args:
filename: A string containing the path to the file.
"""
index = self.repo.index
index.read()
try:
index.add(filename)
index.write()
except Exception as e:
logger.info(
"GitRepo, addfile, Could not add file {}.".format(filename))
logger.debug(e)
def addRemote(self, name, url):
"""Add a remote.
Args:
name: A string containing the name of the remote.
url: A string containing the url to the remote.
"""
try:
self.repo.remotes.create(name, url)
logger.info("Successfully added remote: {} - {}".format(name, url))
except Exception as e:
logger.info("Could not add remote: {} - {}".format(name, url))
logger.debug(e)
try:
self.repo.remotes.set_push_url(name, url)
self.repo.remotes.set_url(name, url)
except Exception as e:
logger.info("Could not set push/fetch urls: {} - {}".format(
name, url))
logger.debug(e)
def checkout(self, commitid):
"""Checkout a commit by a commit id.
Args:
commitid: A string cotaining a commitid.
"""
try:
commit = self.repo.revparse_single(commitid)
self.repo.set_head(commit.oid)
self.repo.reset(commit.oid, GIT_RESET_HARD)
logger.info("Checked out commit: {}".format(commitid))
except Exception as e:
logger.info("Could not check out commit: {}".format(commitid))
logger.debug(e)
def commit(self, message=None):
"""Commit staged files.
Args:
message: A string for the commit message.
Raises:
Exception: If no files in staging area.
"""
if self.isstagingareaclean():
# nothing to commit
return
index = self.repo.index
index.read()
tree = index.write_tree()
try:
author = Signature(self.author_name, self.author_email)
comitter = Signature(self.author_name, self.author_email)
if len(self.repo.listall_reference_objects()) == 0:
# Initial Commit
if message is None:
message = 'Initial Commit from QuitStore'
self.repo.create_commit('HEAD', author, comitter, message,
tree, [])
else:
if message is None:
message = 'New Commit from QuitStore'
self.repo.create_commit('HEAD', author, comitter, message,
tree,
[self.repo.head.get_object().hex])
logger.info('Updates commited')
except Exception as e:
logger.info('Nothing to commit')
logger.debug(e)
if self.gc:
self.garbagecollection()
def commitexists(self, commitid):
"""Check if a commit id is part of the repository history.
Args:
commitid: String of a Git commit id.
Returns:
True, if commitid is part of commit log
False, else.
"""
if commitid in self.getids():
return True
else:
return False
def garbagecollection(self):
"""Start garbage collection.
Args:
commitid: A string cotaining a commitid.
"""
try:
# Check if the garbage collection process is still running
if self.gcProcess is None or self.gcProcess.poll() is not None:
# Start garbage collection with "--auto" option,
# which imidietly terminates, if it is not necessary
self.gcProcess = Popen(["git", "gc", "--auto", "--quiet"],
cwd=self.path)
logger.debug('Spawn garbage collection')
except Exception as e:
logger.debug('Git garbage collection failed to spawn')
logger.debug(e)
def getpath(self):
"""Return the path of the git repository.
Returns:
A string containing the path to the directory of git repo
"""
return self.path
def getcommits(self):
"""Return meta data about exitsting commits.
Returns:
A list containing dictionaries with commit meta data
"""
commits = []
if len(self.repo.listall_reference_objects()) > 0:
for commit in self.repo.walk(self.repo.head.target,
GIT_SORT_REVERSE):
commits.append({
'id':
str(commit.oid),
'message':
str(commit.message),
'commit_date':
datetime.fromtimestamp(
commit.commit_time).strftime('%Y-%m-%dT%H:%M:%SZ'),
'author_name':
commit.author.name,
'author_email':
commit.author.email,
'parents': [c.hex for c in commit.parents],
})
return commits
def getids(self):
"""Return meta data about exitsting commits.
Returns:
A list containing dictionaries with commit meta data
"""
ids = []
if len(self.repo.listall_reference_objects()) > 0:
for commit in self.repo.walk(self.repo.head.target,
GIT_SORT_REVERSE):
ids.append(str(commit.oid))
return ids
def isgarbagecollectionon(self):
"""Return if gc is activated or not.
Returns:
True, if activated
False, if not
"""
return self.gc
def isstagingareaclean(self):
"""Check if staging area is clean.
Returns:
True, if staginarea is clean
False, else.
"""
status = self.repo.status()
for filepath, flags in status.items():
if flags != GIT_STATUS_CURRENT:
return False
return True
def pull(self, remote='origin', branch='master'):
"""Pull if possible.
Return:
True: If successful.
False: If merge not possible or no updates from remote.
"""
try:
self.repo.remotes[remote].fetch()
except Exception as e:
logger.info("Can not pull: Remote {} not found.".format(remote))
logger.debug(e)
ref = 'refs/remotes/' + remote + '/' + branch
remoteid = self.repo.lookup_reference(ref).target
analysis, _ = self.repo.merge_analysis(remoteid)
if analysis & GIT_MERGE_ANALYSIS_UP_TO_DATE:
# Already up-to-date
pass
elif analysis & GIT_MERGE_ANALYSIS_FASTFORWARD:
# fastforward
self.repo.checkout_tree(self.repo.get(remoteid))
master_ref = self.repo.lookup_reference('refs/heads/master')
master_ref.set_target(remoteid)
self.repo.head.set_target(remoteid)
elif analysis & GIT_MERGE_ANALYSIS_NORMAL:
self.repo.merge(remoteid)
tree = self.repo.index.write_tree()
msg = 'Merge from ' + remote + ' ' + branch
author = Signature(self.author_name, self.author_email)
comitter = Signature(self.author_name, self.author_email)
self.repo.create_commit('HEAD', author, comitter, msg, tree,
[self.repo.head.target, remoteid])
self.repo.state_cleanup()
else:
logger.debug('Can not pull. Unknown merge analysis result')
def push(self, remote='origin', branch='master'):
"""Push if possible.
Return:
True: If successful.
False: If diverged or nothing to push.
"""
ref = ['refs/heads/' + branch]
try:
remo = self.repo.remotes[remote]
except Exception as e:
logger.info(
"Can not push. Remote: {} does not exist.".format(remote))
logger.debug(e)
return
try:
remo.push(ref, callbacks=self.callback)
except Exception as e:
logger.info("Can not push to {} with ref {}".format(
remote, str(ref)))
logger.debug(e)
def getRemotes(self):
remotes = {}
try:
for remote in self.repo.remotes:
remotes[remote.name] = [remote.url, remote.push_url]
except Exception as e:
logger.info('No remotes found.')
logger.debug(e)
return {}
return remotes
parser = argparse.ArgumentParser(
description='Run Kiln for every commit on a master branch in a git repo')
parser.add_argument('dir')
parser.add_argument('app_name')
parser.add_argument('data_collector_url')
args = parser.parse_args()
proj_dir = vars(args)['dir']
app_name = vars(args)['app_name']
url = vars(args)['data_collector_url']
repo_path = discover_repository(proj_dir)
if repo_path == None:
raise Exception("Project directory isn't a git repo. Exiting!")
repo = Repository(repo_path)
walker = repo.walk(repo.branches['master'].peel().id,
GIT_SORT_REVERSE | GIT_SORT_TIME)
walker.simplify_first_parent()
all_commits = [x.id for x in walker]
kiln_config_path = os.path.abspath(os.path.join(proj_dir, "kiln.toml"))
for commit in all_commits:
if commit == "":
continue
subprocess.check_call(["git", "reset", "--hard", "HEAD"],
cwd=proj_dir,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
subprocess.check_call(["git", "checkout", str(commit)],
cwd=proj_dir,
stdout=subprocess.DEVNULL,
import pytest
logging.info('Checking if test-metric recording needs to be enabled')
if 'TEST_METRICS_URL' in os.environ:
import datetime
import getpass
import json
import socket
from timeit import default_timer as timer
from pygit2 import Repository
from tests.cook import util
repository_path = os.path.abspath(f'{os.path.dirname(os.path.abspath(__file__))}/../..')
repo = Repository(repository_path)
head = repo.head
commit = repo.revparse_single('HEAD')
git_branch = head.name.replace('refs/heads/', '')
git_commit_hex = commit.hex
elastic_search_url = os.getenv('TEST_METRICS_URL').rstrip('/')
logging.info(f'Sending test metrics to {elastic_search_url}')
@pytest.fixture()
def record_test_metric(request):
start = timer()
yield
try:
end = timer()
now = datetime.datetime.utcnow()
def parse_code_churns(pid, repo_path, branch, start, stop=-1):
"""
Function that is intended to be runned by a process. It extracts the code churns
for a set of commits and stores them in the RES dict.
"""
repo = Repository(repo_path)
head = repo.references.get(branch)
commits = list(
repo.walk(head.target, GIT_SORT_TOPOLOGICAL | GIT_SORT_REVERSE))
start = start - 1 if (start > 0) else start
commits = commits[start:stop] if (stop != -1) else commits[start:]
code_churns = [[] for c in range(len(commits))]
for i, commit in enumerate(tqdm(commits[1:], position=pid)):
diff = repo.diff(commits[i], commit)
tree = commit.tree
patches = [p for p in diff]
stats = diff.stats
# Count the total lines of code and find the biggest file that have been changed
total_tloc = 0
line_of_code_old = 0
for patch in patches:
if patch.delta.is_binary:
continue
new_file = patch.delta.new_file
# Total lines of code
total_tloc += get_file_lines_of_code(repo, tree, new_file)
old_file = patch.delta.old_file
# Total lines of code in the old file
line_of_code_old = max(
line_of_code_old, get_file_lines_of_code(repo, tree, old_file))
# Churned lines of code
cloc = stats.insertions
# Deleted lines of code
dloc = stats.deletions
# Churned files
files_churned = len(patches)
# File count
num_files = count_files(tree, repo)
# Apply relative code churns
measure_one = float(cloc) / total_tloc if (
total_tloc > 0) else float(cloc)
measure_two = float(dloc) / total_tloc if (
total_tloc > 0) else float(cloc)
measure_three = (float(files_churned) / num_files if
(num_files > 0) else float(files_churned))
line_of_code_old = float(line_of_code_old)
# Churn features
code_churns[i].append(str(commit.hex))
code_churns[i].append(str(measure_one))
code_churns[i].append(str(measure_two))
code_churns[i].append(str(measure_three))
code_churns[i].append(str(line_of_code_old))
RES[pid] = code_churns
def do_put(content_path, object_hashes, content, filename):
"""Perform put operation. This is used in the distributed wrapper"""
content_hash = Repository(content_path).create_blob(content)
result = object_hashes[filename] = str(content_hash)
return result
class PyGitEngine(GitContentDatabaseEngine):
def __init__(self, config):
super(PyGitEngine, self).__init__(config)
self.repo = None
def connect(self):
"""Create content directory"""
if not isdir(self.content_path):
init_repository(self.content_path, bare=True)
self.repo = Repository(self.content_path)
self.create_initial_commit()
else:
self.repo = Repository(self.content_path)
@staticmethod
def do_put(content_path, object_hashes, content, filename):
"""Perform put operation. This is used in the distributed wrapper"""
content_hash = Repository(content_path).create_blob(content)
result = object_hashes[filename] = str(content_hash)
return result
def put_attr(self, content, filename):
"""Return attributes for the do_put operation"""
filename = self._inc_name(filename)
return (self.content_path, self.object_hashes, content, filename)
def put(self, content, filename="generic"): # pylint: disable=method-hidden
"""Put content in the content database"""
return self.do_put(*self.put_attr(content, filename))
def get(self, content_hash): # pylint: disable=method-hidden
"""Get content from the content database"""
return_data = self.repo[content_hash].data
return return_data
def find_subhash(self, content_hash):
"""Find hash in git"""
try:
blob = self.repo.revparse_single(content_hash)
return str(blob.id)
except KeyError:
return None
def create_initial_commit(self):
"""Create the initial commit of the git repository"""
empty_tree = self.repo.TreeBuilder().write()
self.create_commit_object(self._initial_message, empty_tree)
def create_commit_object(self, message, tree):
"""Create a commit object"""
references = list(self.repo.references)
master_ref = self.repo.lookup_reference(
self._commit_ref) if len(references) > 0 else None
parents = []
if master_ref is not None:
parents = [master_ref.peel().id]
author = Signature(self._commit_name, self._commit_email)
return self.repo.create_commit(self._commit_ref, author, author,
message, tree, parents)
def new_tree(self, parent):
"""Create new git tree"""
return self.repo.TreeBuilder()
def insert_blob(self, tree, basename, value):
"""Insert blob into tree"""
tree.insert(basename, value, GIT_FILEMODE_BLOB)
def insert_tree(self, tree, basename, value):
"""Insert tree into tree"""
tree.insert(basename, value, GIT_FILEMODE_TREE)
def write_tree(self, tree):
"""Write tree to git directory"""
return tree.write()
default=False, help="Stage and commit all dirty state")
parser.add_argument("--push", dest="wip_push", action="store_true",
default=False, help="Push WIP to default upstream")
parser.add_argument("--force", dest="wip_force", action="store_true",
default=False, help="Force saving WIP (ignore idle time)")
parser.add_argument("--branch-ref", dest="wip_add_branch_name", action="store_true",
default=False, help="Prepend WIP commit message with current branch name")
args = parser.parse_args()
if not is_repo(os.getcwd()):
log_error("Not a git repo")
sys.exit(1)
repo = Repository(os.getcwd() + "/.git")
config = repo.config
remote_url = repo.remotes[args.remote].url
pass_path = None
for glob in credentials_mapping.keys():
if fnmatch.fnmatch(remote_url, glob):
pass_path = credentials_mapping[glob]["target"]
if not args.wip_force and not is_idle_enough("xprintidle-ng"):
sys.exit(0)
diff_size = get_diff_size(repo)
if diff_size == 0:
log_info("no changes to commit")
sys.exit(0)
parser.add_argument("-r", "--git-repo", help="Git repo base directory")
parser.add_argument("-s", "--start-date", help="YYYY-MM-DD format, default to 2016-07-01")
parser.add_argument("-e", "--end-date", help="YYYY-MM-DD format, default to 2016-10-01")
parser.add_argument("-j", "--jira-issues", help="CSV file containing JIRA tickets to correlate commits with (optional)")
args = parser.parse_args()
if not args.git_repo:
raise ValueError("git-repo mandatory")
if not args.start_date:
args.start_date = "2016-07-01"
if not args.end_date:
args.end_date = "2016-10-01"
return args
args = parse_args()
repo = Repository(args.git_repo)
startDate = datetime.strptime(args.start_date, "%Y-%m-%d")
endDate = datetime.strptime(args.end_date, "%Y-%m-%d")
jiraIssuesCsvFilename = args.jira_issues
print "Analyzing repo", repo, "for commits between", startDate, "and", endDate
commits = [commit for commit in repo.walk(repo[repo.head.target].id, GIT_SORT_TOPOLOGICAL | GIT_SORT_TIME | GIT_SORT_REVERSE)]
print "Initial nb of commits: ", len(commits)
commits = filter(GitCommitDateRangeFilter(startDate, endDate).filter, commits)
print "Commits matching date range: ", len(commits)
if jiraIssuesCsvFilename:
jiraKeysExtractor = CsvJiraKeysExtractor(jiraIssuesCsvFilename)
commits = filter(GitCommitJiraFilter(jiraKeysExtractor.extract()).filter, commits)
print "Commits matching JIRA tickets: ", len(commits)
else:
print "No filtering based on JIRA tickets"
def getcurrentrepo():
try:
return Repository(r'.').head.shorthand
except:
return ""
def update_projects(srcdir,
packages,
projects,
other_git,
ws_state,
config,
update_op,
jobs,
dry_run=False,
action="update",
fetch_remote=True):
if (pygit2_features & GIT_FEATURE_HTTPS) == 0:
warning("your libgit2 has no built-in HTTPS support\n")
if (pygit2_features & GIT_FEATURE_SSH) == 0:
warning("your libgit2 has no built-in SSH support\n")
git_remote_callback = GitRemoteCallback(config)
manager = create_multiprocess_manager()
L = manager.Lock()
d = manager.dict()
workload = [(project, project.workspace_path)
for project in projects] + [(None, path) for path in other_git]
result = run_multiprocess_workers(partial(fetch_worker, srcdir,
git_remote_callback,
fetch_remote, dry_run),
workload,
worker_init=fetch_worker_init,
worker_init_args=(L, d),
jobs=jobs,
timeout=900)
done = 0
errors = 0
for r in result:
project, path, e = r
try:
if not e:
repo = Repository(os.path.join(srcdir, path, ".git"))
head_branch, master_branch, tracking_branch = lookup_branches(
repo, project)
if update_op(repo, path, head_branch, master_branch,
tracking_branch):
done += 1
else:
error("cannot fetch remotes for '%s': %s\n" %
(escape(path), escape(str(e))))
errors += 1
except Exception as e:
error("cannot %s '%s': %s\n" %
(action, escape(path), escape(str(e))))
errors += 1
if done:
report = ("%s completed successfully for" %
action.title()) if not dry_run else (
"Dry-run: %s would have been executed for" % action)
if done == 1:
msg("@!%s one repository@|\n" % report)
else:
msg("@!%s %d repositories@|\n" % (report, done))
if errors:
if errors == 1:
warning("an error has occurred\n")
else:
warning("%d errors have occurred\n" % errors)
def __init__(self, repo_path):
self._repo_path = repo_path
self._repo = Repository(self._repo_path)
self._ctx = self._repo[self.revision_id]
def get_code_churns(repo_path, branch):
"""
General function for extracting code churns. It first extracts the code churns for
the first commit and then starts a number of processes(equal to the number of cores
on the computer), which equally extracts the code churns for the remaining commits.
"""
repo = Repository(repo_path)
print(repo)
print(repo.references)
head = repo.references.get(branch)
commits = list(
repo.walk(head.target, GIT_SORT_TOPOLOGICAL | GIT_SORT_REVERSE))
code_churns = [[]]
initial = commits[0]
# Relative code churns
measure_one = 0.0
measure_two = 0.0
measure_three = 1.0
line_of_code_old = 0.0
code_churns[0].append(str(initial.hex))
code_churns[0].append(str(measure_one))
code_churns[0].append(str(measure_two))
code_churns[0].append(str(measure_three))
code_churns[0].append(str(line_of_code_old))
# Check how many processes that could be spawned
cpus = cpu_count()
print("Using {} cpus...".format(cpus))
# Equally split the commit set into the equally sized parts.
quote, remainder = divmod(len(commits), cpus)
processes = [
Process(target=parse_code_churns,
args=(i, repo_path, branch, i * quote + min(i, remainder),
(i + 1) * quote + min(i + 1, remainder)))
for i in range(cpus)
]
for process in processes:
process.start()
start_time = time.time()
for process in processes:
process.join()
end_time = time.time()
print("Done")
print("Overall processing time {}".format(end_time - start_time))
# Assemble the results
churns = []
for _, churn in RES.items():
churns.extend(churn)
churns = list(reversed(churns))
churns.append(code_churns[0])
return churns
def get_head_branch(working_dir):
repo = Repository(working_dir + '.git')
return repo.head.name
def git_repo(cwd):
repo_path = pygit2.discover_repository(cwd)
if repo_path is None:
return
return Repository(repo_path)
def lambda_handler(event, context):
keybucket = event['context']['key-bucket']
outputbucket = event['context']['output-bucket']
pubkey = event['context']['public-key']
### Source IP ranges to allow requests from, if the IP is in one of these the request will not be chacked for an api key
ipranges = []
for i in event['context']['allowed-ips'].split(','):
ipranges.append(ip_network(u'%s' % i))
### APIKeys, it is recommended to use a different API key for each repo that uses this function
apikeys = event['context']['api-secrets'].split(',')
ip = ip_address(event['context']['source-ip'])
secure = False
for net in ipranges:
if ip in net:
secure = True
if 'X-Gitlab-Token' in event['params']['header'].keys():
if event['params']['header']['X-Gitlab-Token'] in apikeys:
secure = True
if 'X-Git-Token' in event['params']['header'].keys():
if event['params']['header']['X-Git-Token'] in apikeys:
secure = True
if 'X-Gitlab-Token' in event['params']['header'].keys():
if event['params']['header']['X-Gitlab-Token'] in apikeys:
secure = True
if 'X-Hub-Signature' in event['params']['header'].keys():
for k in apikeys:
if hmac.new(
str(k), str(event['context']['raw-body']),
hashlib.sha1).hexdigest() == str(
event['params']['header']['X-Hub-Signature'].replace(
'sha1=', '')):
secure = True
if not secure:
logger.error('Source IP %s is not allowed' %
event['context']['source-ip'])
raise Exception('Source IP %s is not allowed' %
event['context']['source-ip'])
try:
repo_name = event['body-json']['project']['path_with_namespace']
except:
repo_name = event['body-json']['repository']['full_name']
try:
remote_url = event['body-json']['project']['git_ssh_url']
except:
try:
remote_url = 'git@' + event['body-json']['repository'][
'links']['html']['href'].replace('https://', '').replace(
'/', ':', 1) + '.git'
except:
remote_url = event['body-json']['repository']['ssh_url']
repo_path = '/tmp/%s' % repo_name
creds = RemoteCallbacks(credentials=get_keys(keybucket, pubkey), )
try:
repository_path = discover_repository(repo_path)
repo = Repository(repository_path)
logger.info('found existing repo, using that...')
except:
logger.info('creating new repo for %s in %s' % (remote_url, repo_path))
repo = create_repo(repo_path, remote_url, creds)
pull_repo(repo, remote_url, creds)
zipfile = zip_repo(repo_path, repo_name)
push_s3(zipfile, repo_name, outputbucket)
if cleanup:
logger.info('Cleanup Lambda container...')
shutil.rmtree(repo_path)
shutil.rm(zipfile)
shutil.rm('/tmp/id_rsa')
shutil.rm('/tmp/id_rsa.pub')
return 'Successfully updated %s' % repo_name
def repository(path):
try:
repo = Repository(path)
except KeyError:
raise JagareError('repo %s not exists' % path)
return repo
class Project1Test(unittest.TestCase):
def __init__(self, test_name, grader):
super(Project1Test, self).__init__(test_name)
self.grader = grader
# setUpClass doesn't work!?
def setUp(self):
self.out_string = ""
self.repo = REPO_TEMPLATE % self.grader.andrewid
os.chdir(self.grader.tmp_dir)
try:
self.tearDown()
except:
pass
self.git_clone(self.grader.root_repo)
os.chdir(self.repo)
self.repository = Repository('.git')
commit = self.resolve_tag()
self.git_checkout(commit.hex)
self.ran = False
self.port = random.randint(1025, 9999)
self.tls_port = random.randint(1025, 9999)
print '\nUsing ports: %d,%d' % (self.port, self.tls_port)
def pAssertEqual(self, arg1, arg2):
try:
self.assertEqual(arg1, arg2)
except Exception as e:
self.print_str(traceback.format_stack()[-2])
raise e
def pAssertTrue(self, test):
try:
self.assertTrue(test)
except Exception as e:
self.print_str(traceback.format_stack()[-2])
raise e
def print_str(self, prt_str):
print(prt_str)
self.out_string += ("\n" + prt_str)
def edit_notes(self, new_note):
notef = self.grader.notes
try:
check_both('cat %s' % (notef), False)
new_note = '\n' + new_note
except:
pass
check_both('echo "%s\nGood." >> %s' % (new_note, notef), False)
check_both('%s %s' % (self.grader.editor, notef))
def confirm(self):
print '-----------------------------------------------'
test = raw_input('OK [y/n]? ').lower() in ['y', '']
self.pAssertTrue(test)
def change_cgi(self, new_path):
self.grader.cgi = new_path
def liso_name(self):
name = './lisod'
# text = raw_input('liso name? ').strip()
# if text: name = text
self.liso_name = name
return name
def get_path(self):
path = None
# text = raw_input('path? ').strip()
# if text: path = text
return path
def get_port(self):
port = self.port
# text = raw_input('port? ').strip()
# if text: port = int(text)
self.port = port
print port
return port
def get_tls_port(self):
tls_port = self.tls_port
# text = raw_input('tls_port? ').strip()
# if text: tls_port = int(text)
self.tls_port = tls_port
print tls_port
return tls_port
def find_path(self, name, tree, path='./', d=0):
if d == 15: return None
name = name.lower().strip()
# bredth first...?
for entry in tree:
if entry.name.lower().strip() == name:
return path
# now check depth...?
entries = [e for e in tree]
for entry in reversed(entries):
obj = self.repository[entry.oid]
if isinstance(obj, Tree):
obj = self.find_path(name, obj, os.path.join(path, entry.name),
d + 1)
if obj:
return obj
return None
def find_file(self, name, tree, d=0):
if d == 15: return None
name = name.lower().strip()
# bredth first...?
for entry in tree:
if entry.name.lower().strip() == name:
resolved = self.repository[entry.oid]
if not isinstance(resolved, Blob):
continue
return resolved
# now check depth...?
entries = [e for e in tree]
for entry in reversed(entries):
obj = self.repository[entry.oid]
if isinstance(obj, Tree):
obj = self.find_file(name, obj, d + 1)
if obj:
return obj
return None
def run_lisod(self, tree):
path = self.get_path()
liso = self.liso_name()
port = self.get_port()
tls_port = self.get_tls_port()
if not path: path = self.find_path('Makefile', tree)
print 'switching to: %s' % path
os.chdir(path)
check_both('make clean', False, False)
check_output('make')
self.ran = True
resource.setrlimit(resource.RLIMIT_STACK,
(resource.RLIM_INFINITY, resource.RLIM_INFINITY))
cmd = '%s %d %d %slisod.log %slisod.lock %s %s %s %s&' % (
liso, port, tls_port, self.grader.tmp_dir, self.grader.tmp_dir,
self.grader.www[:-1], self.grader.cgi, self.grader.priv_key,
self.grader.cert)
#cmd = 'nohup ' + cmd
#cmd = cmd + " > /dev/null"
print cmd
self.pAssertEqual(0, os.system(cmd))
return liso
def git_clone(self, repourl):
with open('/dev/null', 'w') as f:
self.pAssertEqual(
0, check_call(['git', 'clone', repourl], stderr=f, stdout=f))
def git_checkout(self, commit_hex):
with open('/dev/null', 'w') as f:
self.pAssertEqual(
0,
check_call(
['git', 'checkout', '%s' % commit_hex], stdout=f,
stderr=f))
def resolve_tag(self):
try:
tag = self.repository.lookup_reference('refs/tags/checkpoint-%d' %
self.grader.cp_num)
except KeyError:
try:
tag = self.repository.lookup_reference(
'refs/tags/checkpoint_%d' % self.grader.cp_num)
except KeyError:
tag = self.repository.lookup_reference(
'refs/tags/checkpoint%d' % self.grader.cp_num)
#tag = self.repository.lookup_reference('refs/tags/regrade')
commit = self.repository[tag.target]
while isinstance(commit, Tag):
commit = self.repository[commit.target]
return commit
def check_headers(self, response_type, headers, length_content, ext):
self.pAssertEqual(headers['Server'].lower(), 'liso/1.0')
try:
datetime.datetime.strptime(headers['Date'],
'%a, %d %b %Y %H:%M:%S %Z')
except KeyError:
self.print_str('Bad Date header')
except:
self.print_str('Bad Date header: %s' % (headers['Date']))
self.pAssertEqual(int(headers['Content-Length']), length_content)
#self.pAssertEqual(headers['Connection'].lower(), 'close')
if response_type == 'GET' or response_type == 'HEAD':
header_set = set([
'connection', 'content-length', 'date', 'last-modified',
'server', 'content-type'
])
self.pAssertEqual(set(), header_set - set(headers.keys()))
if headers['Content-Type'].lower() != MIME[ext]:
self.print_str('MIME got %s expected %s' %
(headers['Content-Type'].lower(), MIME[ext]))
self.pAssertTrue(
headers['Content-Type'].lower() == MIME[ext]
or headers['Content-Type'].lower() == MIME['.html'])
try:
datetime.datetime.strptime(headers['Last-Modified'],
'%a, %d %b %Y %H:%M:%S %Z')
except:
self.print_str('Bad Last-Modified header: %s' %
(headers['Last-Modified']))
elif response_type == 'POST':
header_set = set(
['connection', 'content-length', 'date', 'server'])
self.pAssertEqual(set(), header_set - set(headers.keys()))
else:
self.fail('Unsupported Response Type...')
# test existence of tag in repo
def test_tag_checkpoint(self):
self.print_str('\n\n----- Testing Tag -----')
self.repository.lookup_reference('refs/tags/checkpoint-%d' %
self.grader.cp_num)
# test turn in timestamp
def test_timestamp(self):
self.print_str('\n\n----- Testing Timestamp -----')
commit = self.resolve_tag()
self.print_str('ref/tags/checkpoint-%d: %s' %
(self.grader.cp_num, commit.hex))
self.print_str('Due: %s' % self.grader.due_date)
utctime = datetime.datetime.utcfromtimestamp(commit.commit_time)
utcoffset = datetime.timedelta(minutes=commit.commit_time_offset)
timestamp = utctime + utcoffset
self.print_str('Timestamp: %s' % timestamp)
timediff = timestamp - self.grader.due_date
if timediff.days >= 0 and\
timediff.seconds > 0 or\
timediff.microseconds > 0:
raise ValueError
# test readme.txt file up to snuff
def test_readme_file(self):
self.print_str('\n\n----- Testing readme.txt file -----')
commit = self.resolve_tag()
tree = commit.tree
print '\n----- readme.txt -----'
readme = self.find_file('readme.txt', tree)
print readme.data,
self.confirm()
self.edit_notes('README:')
# test vulnerabilities.txt up to snuff
def test_vulnerabilities_file(self):
self.print_str('\n\n----- Testing vulnerabilities.txt file -----')
commit = self.resolve_tag()
tree = commit.tree
print '\n----- vulnerabilities.txt -----'
vulnerable = self.find_file('vulnerabilities.txt', tree)
print vulnerable.data,
self.confirm()
self.edit_notes('VULNERABILITIES:')
# test tests.txt up to snuff
def test_tests_file(self):
self.print_str('\n\n----- Testing tests.txt file -----')
commit = self.resolve_tag()
tree = commit.tree
print '\n----- tests.txt -----'
tests = self.find_file('tests.txt', tree)
print tests.data,
self.confirm()
self.edit_notes('TESTS:')
# test Makefile up to snuff
def test_Makefile_file(self):
self.print_str('\n\n----- Testing Makefile file -----')
commit = self.resolve_tag()
tree = commit.tree
print '\n----- Makefile -----'
Makefile = self.find_file('Makefile', tree)
print Makefile.data,
self.confirm()
self.edit_notes('MAKEFILE:')
# test if source up to snuff
def test_inspect_source(self):
self.print_str('\n\n----- Inspect Source cod *.[c|h] -----')
self.print_str(self.grader.source_reminder)
self.pAssertEqual(0, check_call(['bash']))
self.confirm()
self.edit_notes('SOURCE:')
# tests if make properly creates lisod...
def test_lisod_file(self):
self.print_str('\n\n----- Testing make -----')
commit = self.resolve_tag()
path = self.get_path()
if not path: path = self.find_path('Makefile', commit.tree)
os.chdir(path)
check_output('make')
self.pAssertTrue(os.path.exists('./lisod'))
# send all test files to their server
# get output, give 3 second timeout
# check sha's of output
def test_replays(self):
self.print_str('\n\n----- Testing Replays -----')
commit = self.resolve_tag()
self.run_lisod(commit.tree)
time.sleep(3)
replays_dir = os.path.join(self.grader.tmp_dir, 'replays')
if not os.path.exists(replays_dir):
os.makedirs(replays_dir)
files = os.listdir(replays_dir)
num_passed = 0
num_files = 0
for fname in files:
basename, extension = os.path.splitext(fname)
if extension == '.test':
num_files += 1
self.print_str('testing %s...' % fname)
fname = os.path.join(self.grader.tmp_dir + 'replays', fname)
outfile = os.path.join(self.grader.tmp_dir + 'replays',
'%s_%s.out' % (basename, self.repo))
command = 'ncat -i 1s localhost %d < %s > %s' % (
self.port, fname, outfile)
check_both(command, False, False)
with open(
os.path.join(self.grader.tmp_dir + 'replays',
basename + '.out')) as f:
with open(outfile) as f2:
outhash = hashlib.sha256(f.read()).hexdigest()
out2hash = hashlib.sha256(f2.read()).hexdigest()
if outhash == out2hash:
self.print_str('ok')
num_passed += 1
else:
self.print_str('failed')
check_both('rm %s' % outfile)
self.print_str('passed %d of %d' % (num_passed, num_files))
self.pAssertEqual(num_passed, num_files)
def test_HEAD_headers(self):
self.print_str('----- Testing Headers -----')
tests = {
'http://127.0.0.1:%d/index.html':
('f5cacdcb48b7d85ff48da4653f8bf8a7c94fb8fb43407a8e82322302ab13becd',
802),
'http://127.0.0.1:%d/images/liso_header.png':
('abf1a740b8951ae46212eb0b61a20c403c92b45ed447fe1143264c637c2e0786',
17431),
'http://127.0.0.1:%d/style.css':
('575150c0258a3016223dd99bd46e203a820eef4f6f5486f7789eb7076e46736a',
301)
}
commit = self.resolve_tag()
self.git_checkout(commit.hex)
name = self.run_lisod(commit.tree)
time.sleep(1)
for test in tests:
root, ext = os.path.splitext(test)
response = requests.head(test % self.port, timeout=10.0)
self.check_headers(response.request.method, response.headers,
tests[test][1], ext)
def test_HEAD(self):
self.print_str('----- Testing HEAD -----')
tests = {
'http://127.0.0.1:%d/index.html':
('f5cacdcb48b7d85ff48da4653f8bf8a7c94fb8fb43407a8e82322302ab13becd',
802),
'http://127.0.0.1:%d/images/liso_header.png':
('abf1a740b8951ae46212eb0b61a20c403c92b45ed447fe1143264c637c2e0786',
17431),
'http://127.0.0.1:%d/style.css':
('575150c0258a3016223dd99bd46e203a820eef4f6f5486f7789eb7076e46736a',
301)
}
commit = self.resolve_tag()
self.git_checkout(commit.hex)
name = self.run_lisod(commit.tree)
time.sleep(1)
for test in tests:
root, ext = os.path.splitext(test)
response = requests.head(test % self.port, timeout=10.0)
contenthash = hashlib.sha256(response.content).hexdigest()
self.pAssertEqual(200, response.status_code)
def test_GET(self):
self.print_str('----- Testing GET -----')
tests = {
'http://127.0.0.1:%d/index.html':
'f5cacdcb48b7d85ff48da4653f8bf8a7c94fb8fb43407a8e82322302ab13becd',
'http://127.0.0.1:%d/images/liso_header.png':
'abf1a740b8951ae46212eb0b61a20c403c92b45ed447fe1143264c637c2e0786',
'http://127.0.0.1:%d/style.css':
'575150c0258a3016223dd99bd46e203a820eef4f6f5486f7789eb7076e46736a'
}
commit = self.resolve_tag()
self.git_checkout(commit.hex)
name = self.run_lisod(commit.tree)
time.sleep(1)
for test in tests:
root, ext = os.path.splitext(test)
response = requests.get(test % self.port, timeout=10.0)
contenthash = hashlib.sha256(response.content).hexdigest()
self.pAssertEqual(200, response.status_code)
self.pAssertEqual(contenthash, tests[test])
def test_POST(self):
self.print_str('----- Testing POST -----')
tests = {
'http://127.0.0.1:%d/index.html':
'f5cacdcb48b7d85ff48da4653f8bf8a7c94fb8fb43407a8e82322302ab13becd',
}
commit = self.resolve_tag()
self.git_checkout(commit.hex)
name = self.run_lisod(commit.tree)
time.sleep(1)
for test in tests:
root, ext = os.path.splitext(test)
# for checkpoint 2, this should time out; we told them to swallow the data and ignore
try:
response = requests.post(test % self.port,
data='dummy data',
timeout=3.0)
#except requests.exceptions.Timeout:
except requests.exceptions.RequestException:
print 'timeout'
continue
except socket.timeout:
print 'socket.timeout'
continue
# if they do return something, make sure it's OK
self.pAssertEqual(200, response.status_code)
def test_bw(self):
print '(----- Testing BW -----'
check_output('echo "----- Testing BW ----" >> %s' %
self.grader.results)
commit = self.resolve_tag()
self.git_checkout(commit.hex)
name = self.run_lisod(commit.tree)
time.sleep(1)
self.pAssertEqual(
0,
os.system(
'curl -m 10 -o /dev/null http://127.0.0.1:%d/big.html 2>> %s' %
(self.port, self.grader.results)))
def tearDown(self):
#check_both('rm ' + self.grader.tmp_dir + 'lisod.log', False, False)
check_both('rm ' + self.grader.tmp_dir + 'lisod.lock', False, False)
os.chdir(self.grader.tmp_dir)
shutil.rmtree(self.repo)
if sys.exc_info() == (None, None, None): #test succeeded
self.out_string += '\nok'
else:
self.out_string += '\nfailed'
if self.out_string:
check_both('echo "%s" >> %s' %
(self.out_string, self.grader.results))
if self.ran:
print 'trying "killall -9 %s"' % os.path.basename(self.liso_name)
check_both('killall -9 %s' % os.path.basename(self.liso_name),
True, False)
def test_write_tree_to(self):
pygit2.option(pygit2.GIT_OPT_ENABLE_STRICT_OBJECT_CREATION, False)
with utils.TemporaryRepository(('tar', 'emptyrepo')) as path:
nrepo = Repository(path)
id = self.repo.index.write_tree(nrepo)
assert nrepo[id] is not None
""" Handles all things git (there aren't many of them) """ from pygit2 import discover_repository, Repository import os PATH = discover_repository(os.path.dirname(__file__)) REPO = Repository(PATH) # https://www.pygit2.org/repository.html
import itertools
parser = argparse.ArgumentParser(description='Run Kiln for every commit on the default branch in a git repo')
parser.add_argument('dir')
parser.add_argument('app_name')
parser.add_argument('data_collector_url')
args = parser.parse_args()
proj_dir = vars(args)['dir']
app_name = vars(args)['app_name']
url = vars(args)['data_collector_url']
repo_path = discover_repository(proj_dir)
if repo_path == None:
raise Exception("Project directory isn't a git repo. Exiting!")
repo = Repository(repo_path)
default_branch = repo.resolve_refish('origin/HEAD')[1].shorthand
walker = repo.walk(repo.branches[default_branch].peel().id, GIT_SORT_REVERSE | GIT_SORT_TIME)
walker.simplify_first_parent()
all_commits = [x.id for x in walker]
kiln_config_path = os.path.abspath(os.path.join(proj_dir, "kiln.toml"))
for commit in all_commits:
if commit == "":
continue
subprocess.check_call(["git", "reset", "--hard", "HEAD"], cwd=proj_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.check_call(["git", "checkout", str(commit)], cwd=proj_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
if not os.path.exists(os.path.abspath(os.path.join(proj_dir, "Gemfile.lock"))):
print("No Gemfile.lock, skipping commit ", str(commit))
continue
import sys
import os
import io
import re
import time
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
from distutils.sysconfig import customize_compiler
from distutils.command.build_py import build_py
version = '1.4'
branchversion = version
try:
from pygit2 import Repository
head = Repository('.').head.shorthand
if head != 'master':
branchversion = version + '.' + head
except:
Repository = None
print('cmf', branchversion)
# Try to import numpy, if it fails we have a problem
try:
# Import a function to get a path to the include directories of numpy
# noinspection PyPackageRequirements
from numpy import get_include as get_numpy_include
except ImportError:
raise RuntimeError(
"For building and running of cmf an installation of numpy is needed")
def switch_branch(working_dir, branch_name):
repo = Repository(working_dir + '.git')
print('switch to: ' + branch_name)
branch = repo.branches[branch_name]
repo.checkout('refs/heads/' + branch.branch_name)
def post(event, context):
# Logging the entire event is a cheap simple way to make debugging easier
# Often times just being able to see the event information quickly can help
# Troubleshoot an issue faster than hooking up a debugger
logger.info(event)
# We always want to take the shortest path through our functions. Check for anything fatal first.
try:
output_bucket = os.environ['output_bucket']
except:
raise Exception(
'Output Bucket not defined. Set the environment variable for the funcion'
)
# Here we take a few steps to get the JSON into the body object
# If this came in as a proxy request, or a direct API Gateway request
# or a boto3 invokation the format of the body could be a few different types
# With this stepped approach we can gaurantee that no matter how this was caled
# we will have JSON in the body variable.
if "body" in event:
body = json.loads(event['body'])
else:
try:
body = json.loads(event)
except:
body = event
# We will still validate this before doing anything with it, but if we are missing
# any essential components we should end early to save processing time.
# No point in computing hashes for a payload that is missing data we need.
try:
full_name = body['repository']['full_name']
except KeyError:
raise Exception('Failed to find full_name in json post body')
try:
remote_url = body['repository']['clone_url']
except KeyError:
raise Exception('Failed to find clone_url name in json post body')
try:
github_secrets = os.environ['github_secrets']
except:
raise Exception(
'Github secrets not defined. Set the environment variable for the funcion'
)
if "headers" in event and "X-GitHub-Event" in event['headers']:
# We only care about push events, if this isn't one politely exit
if event['headers']['X-GitHub-Event'] != "push":
return {
"statusCode": 200,
"body": json.dumps('Skipping - Not a push event')
}
# We split this env variable because we could be re-using this function for multiple API
# endpoints, multiple repos etc. It is best practice to have a secret per repo
# so even if we use this exact endpoint we can still feed it multiple repos with multiple
# keys. We define each key with a , to separate them.
apikeys = github_secrets.split(',')
# set a validation key, we will check multiple keys so it holds our result
secure = False
# Compute out the hash and validate the signature. If it passes set secure, otherwise throw an error
if 'X-Hub-Signature' in event['headers'].keys():
signature = event['headers']['X-Hub-Signature']
for k in apikeys:
computed_hash = hmac.new(k.encode('ascii'),
event['body'].encode('ascii'),
hashlib.sha1)
computed_signature = '='.join(['sha1', computed_hash.hexdigest()])
hmac.compare_digest(computed_signature.encode('ascii'),
signature.encode('ascii'))
if hmac.compare_digest(computed_signature.encode('ascii'),
signature.encode('ascii')):
secure = True
if secure == False:
raise Exception('Failed to validate authenticity of webhook message')
repo_name = full_name + '/branch/' + branch_name
repo_path = '/tmp/%s' % repo_name
# If we have an existing repo (if this function is still warm / is not a cold start)
# we can re-use that repo on the file system and update it to save us some time and bandwidth
try:
repository_path = discover_repository(repo_path)
repo = Repository(repository_path)
logger.info('found existing repo, using that...')
# If a previous repo is not found we will create it
except Exception:
logger.info('creating new repo for %s in %s' % (remote_url, repo_path))
repo = create_repo(repo_path, remote_url)
# Re-used or created, we now have a repo reference to pull against
pull_repo(repo, branch_name, remote_url)
# Compile the site to our pre-defined path
build_hugo(repo_path, build_path)
# Sync the site to our public s3 bucket for hosting
upload_to_s3(build_path, output_bucket)
if cleanup:
logger.info('Cleanup Lambda container...')
shutil.rmtree(repo_path)
# We have to return a status code otherwise the API Gateway will give a server error
# however we are likely exceeding the 29s hard timeout limit on the API gateway
# but if we can return correctly we should attempt to, that window could be changed later
# or we could execute in time occassionally
return {
"statusCode": 200,
"body": json.dumps('Successfully updated %s' % repo_name)
}
def delete_branch(working_dir, branch_name):
repo = Repository(working_dir + '.git')
print branch_name
repo.branches.delete(branch_name)
def check(dir):
repo = Repository(dir + '.git')
print('Path to the git repository is:', repo.path)
if not repo.is_bare:
print('Working directory of the repository is', repo.workdir)
return repo
def show_status(srcdir,
packages,
projects,
other_git,
ws_state,
show_up_to_date=True,
cache=None):
def create_upstream_status(repo, head_branch, master_branch,
master_remote_branch, tracking_branch):
status = []
if not repo.head_is_detached and not has_pending_merge(repo):
if tracking_branch is not None:
if master_remote_branch is not None:
if tracking_branch.remote_name != master_remote_branch.remote_name:
status.append("@!@{rf}remote '%s'" %
tracking_branch.remote_name)
if need_push(repo, head_branch):
status.append("@!@{yf}needs push")
elif need_pull(repo, head_branch):
status.append("@!@{cf}needs pull")
elif not is_up_to_date(repo, head_branch):
status.append("@!@{yf}needs pull -M")
else:
if head_branch:
status.append("@!on branch '%s'" % repo.head.shorthand)
else:
status.append("empty branch")
if master_remote_branch is None:
status.append("@!@{rf}no remote")
elif master_branch is None:
status.append("@!@{rf}untracked remote")
if is_up_to_date(repo, master_branch) or need_push(
repo, master_branch):
if need_pull(repo, head_branch, master_branch):
status.append("@!@{cf}needs pull -L")
else:
if not is_ancestor(repo, master_branch, head_branch):
status.append("@!@{yf}needs merge --from-master")
if not is_up_to_date(repo, head_branch, master_branch):
status.append("@!@{yf}needs merge --to-master")
if master_branch is not None and master_remote_branch is not None and (
tracking_branch is None
or tracking_branch.name != master_remote_branch.name):
if need_push(repo, master_branch):
status.append("@!@{yf}%s needs push" %
master_branch.shorthand)
elif need_pull(repo, master_branch):
status.append("@!@{cf}%s needs pull" %
master_branch.shorthand)
elif not is_up_to_date(repo, master_branch):
status.append("@!@{yf}%s needs merge" %
master_branch.shorthand)
return status
def create_local_status(repo, upstream_status, is_dirty):
status = []
if repo.head_is_detached:
status.append("@!@{rf}detached HEAD")
return status
if has_pending_merge(repo):
if repo.index.conflicts:
status.append("@!@{rf}merge conflicts")
else:
status.append("@!@{yf}merged, needs commit")
return status
if is_dirty:
status.append("@!@{yf}needs commit")
status += upstream_status
if not status:
if not show_up_to_date:
return None
status.append("@!@{gf}up-to-date")
return status
table = TableView("Package", "Path", "Status")
found_packages = set()
for project in projects:
repo = Repository(os.path.join(srcdir, project.workspace_path, ".git"))
dirty_files = [
a for a, b in iteritems(repo.status())
if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT
]
head_branch = get_head_branch(repo)
tracking_branch = head_branch.upstream if head_branch else None
master_remote = get_origin(repo, project)
if master_remote is not None:
master_remote_branch = repo.lookup_branch(
"%s/%s" % (master_remote.name, project.master_branch),
GIT_BRANCH_REMOTE)
master_branch = None
if master_remote_branch is not None:
for name in repo.listall_branches(GIT_BRANCH_LOCAL):
b = repo.lookup_branch(name, GIT_BRANCH_LOCAL)
if b.upstream and b.upstream.branch_name == master_remote_branch.branch_name:
master_branch = b
break
else:
master_remote_branch = None
master_branch = None
ws_packages = find_catkin_packages(srcdir,
project.workspace_path,
cache=cache)
found_packages |= set(ws_packages.keys())
upstream_status = create_upstream_status(repo, head_branch,
master_branch,
master_remote_branch,
tracking_branch)
for name, pkg_list in iteritems(ws_packages):
if name not in packages:
continue
for pkg in pkg_list:
is_dirty = False
local_path = os.path.relpath(pkg.workspace_path,
project.workspace_path)
if dirty_files and local_path == ".":
is_dirty = True
else:
for fpath in dirty_files:
if path_has_prefix(fpath, local_path):
is_dirty = True
break
status = create_local_status(repo, upstream_status, is_dirty)
if status is not None:
head, tail = os.path.split(pkg.workspace_path)
pkg_path = escape(head + "/" if tail ==
name else pkg.workspace_path)
table.add_row(escape(name), pkg_path, status)
for path in other_git:
repo = Repository(os.path.join(srcdir, path, ".git"))
dirty_files = [
a for a, b in iteritems(repo.status())
if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT
]
head_branch = get_head_branch(repo)
tracking_branch = head_branch.upstream if head_branch else None
ws_packages = find_catkin_packages(srcdir, path, cache=cache)
found_packages |= set(ws_packages.keys())
upstream_status = create_upstream_status(repo, head_branch, None, None,
tracking_branch)
for name, pkg_list in iteritems(ws_packages):
if name not in packages:
continue
for pkg in pkg_list:
is_dirty = False
local_path = os.path.relpath(pkg.workspace_path, path)
if dirty_files and local_path == ".":
is_dirty = True
else:
for fpath in dirty_files:
if path_has_prefix(fpath, local_path):
is_dirty = True
break
status = create_local_status(repo, upstream_status, is_dirty)
if status is not None:
head, tail = os.path.split(pkg.workspace_path)
pkg_path = escape(head + "/" if tail ==
name else pkg.workspace_path)
table.add_row(escape(name), pkg_path, status)
missing = set(packages) - found_packages
for name in missing:
path_list = []
status = "no git"
if name in ws_state.ws_packages:
for pkg in ws_state.ws_packages[name]:
if not os.path.isdir(os.path.join(srcdir, pkg.workspace_path)):
status = "@{rf}deleted"
head, tail = os.path.split(pkg.workspace_path)
path_list.append(
escape(head + "/" if tail == name else pkg.workspace_path))
table.add_row(escape(name), path_list, status)
if table.empty():
if found_packages:
msg("Everything is @!@{gf}up-to-date@|.\n")
else:
warning("no Git repositories\n")
else:
table.sort(0)
table.write(sys.stdout)
def __init__(self, buffer_id, config_dir, directory):
super(GitViewerWidget, self).__init__()
repo_path = discover_repository(directory)
if repo_path:
repo = Repository(repo_path)
commit_size = 0
lastest_commit = None
for commit in repo.walk(repo.head.target, GIT_SORT_TOPOLOGICAL):
commit_size += 1
if not lastest_commit:
lastest_commit = commit
remote_branch_size = 0
for branch in repo.branches.remote:
if branch != "origin/HEAD" and branch != "origin/master":
remote_branch_size += 1
# Change background.
self.setStyleSheet("background-color: #000000")
# Add main box.
main_box = QVBoxLayout()
main_box.setSpacing(0)
main_box.setContentsMargins(0, 0, 0, 0)
self.setLayout(main_box)
# Add repo top area.
self.repo_top_font_size = 18
self.repo_top_area = QWidget()
self.repo_top_layout = QVBoxLayout()
self.repo_top_layout.setSpacing(0)
self.repo_top_layout.setContentsMargins(30, 30, 30, 30)
self.repo_top_area.setLayout(self.repo_top_layout)
main_box.addWidget(self.repo_top_area)
# Add repo title.
self.repo_title = QLabel(repo.path)
self.repo_title.setStyleSheet("QLabel {color: #E98123;}")
self.repo_title.setFont(QFont('Arial', 20))
self.repo_top_layout.addWidget(self.repo_title)
# Add summary info.
self.repo_summary_area = QWidget()
self.repo_summary_layout = QHBoxLayout()
self.repo_summary_layout.setSpacing(30)
self.repo_summary_layout.setContentsMargins(0, 0, 0, 0)
self.repo_summary_area.setLayout(self.repo_summary_layout)
self.repo_top_layout.addWidget(self.repo_summary_area)
# Add head info.
self.head_info = QLabel("{} {} {} {}".format(
repo.head.shorthand,
str(remote_branch_size) + " branches",
str(commit_size) + " commits",
format_bytes(get_dir_size(repo_path))))
self.head_info.setStyleSheet("QLabel {color: #C46C1D;}")
self.head_info.setFont(QFont('Arial', self.repo_top_font_size))
self.repo_summary_layout.addWidget(self.head_info)
self.repo_summary_layout.addStretch(1)
# Add lastest commit info.
self.lastest_commit_area = QWidget()
self.lastest_commit_layout = QHBoxLayout()
self.lastest_commit_layout.setSpacing(30)
self.lastest_commit_layout.setContentsMargins(0, 30, 0, 30)
self.lastest_commit_area.setLayout(self.lastest_commit_layout)
self.repo_top_layout.addWidget(self.lastest_commit_area)
self.lastest_commit_info = QLabel(
"Lastest: {} {}... {} {}".format(
lastest_commit.author.name,
lastest_commit.message.split("\n")[0][:40],
lastest_commit.hex[:7],
datetime.utcfromtimestamp(
lastest_commit.author.time).strftime(
'%Y-%m-%d %H:%M:%S')))
self.lastest_commit_info.setStyleSheet("QLabel {color: #6C6C6C;}")
self.lastest_commit_info.setFont(
QFont('Arial', self.repo_top_font_size))
self.lastest_commit_layout.addWidget(self.lastest_commit_info)
self.lastest_commit_layout.addStretch(1)
# Add info box.
info_box = QWidget()
info_area_layout = QHBoxLayout()
info_area_layout.setSpacing(30)
info_area_layout.setContentsMargins(30, 0, 30, 30)
info_box.setLayout(info_area_layout)
# Add category panel.
category_panel_listview = QListView()
category_panel_listview.setSpacing(10)
category_panel_listview.setStyleSheet(
"QListView {font-size: 40px;}")
category_panel_model = QStringListModel()
category_panel_list = ["Status", "Commit", "Branch", "Submodule"]
category_panel_model.setStringList(category_panel_list)
category_panel_listview.setModel(category_panel_model)
info_area_layout.addWidget(category_panel_listview)
info_area_layout.setStretchFactor(category_panel_listview, 1)
# Add view panel.
view_panel_listview = QListView()
view_panel_listview.setSpacing(10)
view_panel_listview.setStyleSheet("QListView {font-size: 40px;}")
view_panel_model = QStringListModel()
view_panel_list = ["Current repo update to date."]
view_panel_model.setStringList(view_panel_list)
view_panel_listview.setModel(view_panel_model)
info_area_layout.addWidget(view_panel_listview)
info_area_layout.setStretchFactor(view_panel_listview, 4)
# Add help panel.
help_panel_listview = QListView()
help_panel_listview.setSpacing(10)
help_panel_listview.setStyleSheet("QListView {font-size: 40px;}")
help_panel_model = QStringListModel()
help_panel_list = [
"Press l: Git pull", "Press u: Git push",
"Press a: Commit all", "Press s: Commit file",
"Press k: Cancel file"
]
help_panel_model.setStringList(help_panel_list)
help_panel_listview.setModel(help_panel_model)
info_area_layout.addWidget(help_panel_listview)
info_area_layout.setStretchFactor(help_panel_listview, 1)
main_box.addWidget(info_box)