def createTask(taskId, gitKey, gitAddress):
"""
fortify 命令行调用
fortify 使用手册 HPE_SCA_Guide_16.20.pdf
:param taskId:
:param gitKey:
:param gitAddress:
:return:
"""
clonePath = os.path.join(BASE_DIR, "gitClone")
outputPath = os.path.join(BASE_DIR, "reports")
if not os.path.exists(clonePath):
os.mkdir(clonePath)
if not os.path.exists(outputPath):
os.mkdir(outputPath)
fortifyLogMsg(taskId=taskId, msg=u"拉取代码" + gitAddress, code=1)
localPath = gitAddress.replace("http://", "").replace("https://", "").replace("/", "_").split("?")[0]
localPath = localPath.replace("git@", "").replace(":", "_").replace(".git", "")
localPath = os.path.join(clonePath, localPath)
buildName = "_".join(localPath.split("_")[1:])
try:
if gitKey and gitAddress:
from gittle.auth import GittleAuth
# https://github.com/FriendCode/gittle/issues/18
authkey = GittleAuth(pkey=gitKey)
if not os.path.exists(localPath):
Gittle.clone(gitAddress, local_path=localPath, auth=authkey)
else:
repo = Gittle(localPath, origin_uri=gitAddress, auth=authkey)
repo.pull()
else:
if not os.path.exists(localPath):
Gittle.clone(gitAddress, localPath)
else:
repo = Gittle(localPath, gitAddress)
repo.pull()
except Exception, e:
# raise e
fortifyLogMsg(taskId, u"任务拉取代码遇到异常" + str(e.message), "", code=2)
return "" # 结束任务
class Gittle(object):
"""All paths used in Gittle external methods must be paths relative to the git repository
"""
DEFAULT_COMMIT = "HEAD"
DEFAULT_BRANCH = "master"
DEFAULT_REMOTE = "origin"
DEFAULT_MESSAGE = "**No Message**"
DEFAULT_USER_INFO = {"name": None, "email": None}
DIFF_FUNCTIONS = {
"classic": utils.git.classic_tree_diff,
"dict": utils.git.dict_tree_diff,
"changes": utils.git.dict_tree_diff,
}
DEFAULT_DIFF_TYPE = "dict"
HIDDEN_REGEXES = [
# Hide git directory
r".*\/\.git\/.*"
]
# References
REFS_BRANCHES = "refs/heads/"
REFS_REMOTES = "refs/remotes/"
REFS_TAGS = "refs/tags/"
# Name pattern truths
# Used for detecting if files are :
# - deleted
# - added
# - changed
PATTERN_ADDED = (False, True)
PATTERN_REMOVED = (True, False)
PATTERN_MODIFIED = (True, True)
# Permissions
MODE_DIRECTORY = 040000 # Used to tell if a tree entry is a directory
# Tree depth
MAX_TREE_DEPTH = 1000
# Acceptable Root paths
ROOT_PATHS = (os.path.curdir, os.path.sep)
def __init__(self, repo_or_path, origin_uri=None, auth=None, report_activity=None, *args, **kwargs):
if isinstance(repo_or_path, DulwichRepo):
self.repo = repo_or_path
elif isinstance(repo_or_path, Gittle):
self.repo = DulwichRepo(repo_or_path.path)
elif isinstance(repo_or_path, basestring):
path = os.path.abspath(repo_or_path)
self.repo = DulwichRepo(path)
else:
logging.warning("Repo is of type %s" % type(repo_or_path))
raise Exception("Gittle must be initialized with either a dulwich repository or a string to the path")
# Set path
self.path = self.repo.path
# The remote url
self.origin_uri = origin_uri
# Report client activty
self._report_activity = report_activity
# Build ignore filter
self.hidden_regexes = copy.copy(self.HIDDEN_REGEXES)
self.hidden_regexes.extend(self._get_ignore_regexes())
self.ignore_filter = utils.paths.path_filter_regex(self.hidden_regexes)
self.filters = [self.ignore_filter]
# Get authenticator
if auth:
self.authenticator = auth
else:
self.auth(*args, **kwargs)
def report_activity(self, *args, **kwargs):
if not self._report_activity:
return
return self._report_activity(*args, **kwargs)
def _format_author(self, name, email):
return "%s <%s>" % (name, email)
def _format_userinfo(self, userinfo):
name = userinfo.get("name")
email = userinfo.get("email")
if name and email:
return self._format_author(name, email)
return None
def _format_ref(self, base, extra):
return "".join([base, extra])
def _format_ref_branch(self, branch_name):
return self._format_ref(self.REFS_BRANCHES, branch_name)
def _format_ref_remote(self, remote_name):
return self._format_ref(self.REFS_REMOTES, remote_name)
def _format_ref_tag(self, tag_name):
return self._format_ref(self.REFS_TAGS, tag_name)
@property
def head(self):
"""Return SHA of the current HEAD
"""
return self.repo.head()
@property
def is_bare(self):
"""Bare repositories have no working directories or indexes
"""
return self.repo.bare
@property
def is_working(self):
return not (self.is_bare)
def has_index(self):
"""Opposite of is_bare
"""
return self.repo.has_index()
@property
def has_commits(self):
"""
If the repository has no HEAD we consider that is has no commits
"""
try:
self.repo.head()
except KeyError:
return False
return True
def ref_walker(self, ref=None):
"""
Very simple, basic walker
"""
ref = ref or "HEAD"
sha = self._commit_sha(ref)
return self.repo.revision_history(sha)
def branch_walker(self, branch):
branch = branch or self.DEFAULT_BRANCH
ref = self._format_ref_branch(branch)
return self.ref_walker(ref)
def commit_info(self, start=0, end=None, branch=None):
"""Return a generator of commits with all their attached information
"""
if not self.has_commits:
return []
commits = [utils.git.commit_info(entry) for entry in self.branch_walker(branch)]
if not end:
return commits
return commits[start:end]
@funky.uniquify
def recent_contributors(self, n=None, branch=None):
n = n or 10
return funky.pluck(self.commit_info(end=n, branch=branch), "author")
@property
def commit_count(self):
try:
return len(self.ref_walker())
except KeyError:
return 0
def commits(self):
"""Return a list of SHAs for all the concerned commits
"""
return [commit["sha"] for commit in self.commit_info()]
@property
def git_dir(self):
return self.repo.controldir()
def auth(self, *args, **kwargs):
self.authenticator = GittleAuth(*args, **kwargs)
return self.authenticator
# Generate a branch selector (used for pushing)
def _wants_branch(self, branch_name=None):
branch_name = branch_name or self.DEFAULT_BRANCH
refs_key = self._format_ref_branch(branch_name)
sha = self.branches[branch_name]
def wants_func(old):
refs_key = self._format_ref_branch(branch_name)
return {refs_key: sha}
return wants_func
def _get_ignore_regexes(self):
gitignore_filename = os.path.join(self.path, ".gitignore")
if not os.path.exists(gitignore_filename):
return []
lines = open(gitignore_filename).readlines()
globers = map(lambda line: line.rstrip(), lines)
return utils.paths.globers_to_regex(globers)
# Get the absolute path for a file in the git repo
def abspath(self, repo_file):
return os.path.abspath(os.path.join(self.path, repo_file))
# Get the relative path from the absolute path
def relpath(self, abspath):
return os.path.relpath(abspath, self.path)
@property
def last_commit(self):
return self[self.repo.head()]
@property
def index(self):
return self.repo.open_index()
@classmethod
def init(cls, path, bare=None, *args, **kwargs):
"""Initialize a repository"""
mkdir_safe(path)
# Constructor to use
if bare:
constructor = DulwichRepo.init_bare
else:
constructor = DulwichRepo.init
# Create dulwich repo
repo = constructor(path)
# Create Gittle repo
return cls(repo, *args, **kwargs)
@classmethod
def init_bare(cls, *args, **kwargs):
kwargs.setdefault("bare", True)
return cls.init(*args, **kwargs)
def get_client(self, origin_uri=None, **kwargs):
# Get the remote URL
origin_uri = origin_uri or self.origin_uri
# Fail if inexistant
if not origin_uri:
raise InvalidRemoteUrl()
client_kwargs = {}
auth_kwargs = self.authenticator.kwargs()
client_kwargs.update(auth_kwargs)
client_kwargs.update(kwargs)
client_kwargs.update({"report_activity": self.report_activity})
client, remote_path = get_transport_and_path(origin_uri, **client_kwargs)
return client, remote_path
def push_to(self, origin_uri, branch_name=None, progress=None, progress_stderr=None):
selector = self._wants_branch(branch_name=branch_name)
client, remote_path = self.get_client(origin_uri, progress_stderr=progress_stderr)
return client.send_pack(remote_path, selector, self.repo.object_store.generate_pack_contents, progress=progress)
# Like: git push
def push(self, origin_uri=None, branch_name=None, progress=None, progress_stderr=None):
return self.push_to(origin_uri, branch_name, progress, progress_stderr)
# Not recommended at ALL ... !!!
def dirty_pull_from(self, origin_uri, branch_name=None):
# Remove all previously existing data
rmtree(self.path)
mkdir_safe(self.path)
self.repo = DulwichRepo.init(self.path)
# Fetch brand new copy from remote
return self.pull_from(origin_uri, branch_name)
def pull_from(self, origin_uri, branch_name=None):
return self.fetch(origin_uri)
# Like: git pull
def pull(self, origin_uri=None, branch_name=None):
return self.pull_from(origin_uri, branch_name)
def fetch_remote(self, origin_uri=None):
# Get client
client, remote_path = self.get_client(origin_uri=origin_uri)
# Fetch data from remote repository
remote_refs = client.fetch(remote_path, self.repo)
return remote_refs
def _setup_fetched_refs(self, refs, origin, bare):
remote_tags = utils.git.subrefs(refs, "refs/tags")
remote_heads = utils.git.subrefs(refs, "refs/heads")
# Filter refs
clean_remote_tags = utils.git.clean_refs(remote_tags)
clean_remote_heads = utils.git.clean_refs(remote_heads)
# Base of new refs
heads_base = "refs/remotes/" + origin
if bare:
heads_base = "refs/heads"
# Import branches
self.import_refs(heads_base, clean_remote_heads)
# Import tags
self.import_refs("refs/tags", clean_remote_tags)
# Update HEAD
self["HEAD"] = refs["HEAD"]
def fetch(self, origin_uri=None, bare=None, origin=None):
bare = bare or False
origin = origin or self.DEFAULT_REMOTE
# Remote refs
remote_refs = self.fetch_remote(origin_uri)
# Update head
# Hit repo because head doesn't yet exist so
# print("REFS = %s" % remote_refs)
# Update refs (branches, tags, HEAD)
self._setup_fetched_refs(remote_refs, origin, bare)
# Checkout working directories
if not bare:
self.checkout_all()
else:
self.update_server_info()
@classmethod
def clone(cls, origin_uri, local_path, auth=None, mkdir=True, bare=False, *args, **kwargs):
"""Clone a remote repository"""
mkdir_safe(local_path)
# Initialize the local repository
if bare:
local_repo = cls.init_bare(local_path)
else:
local_repo = cls.init(local_path)
repo = cls(local_repo, origin_uri=origin_uri, auth=auth, *args, **kwargs)
repo.fetch(bare=bare)
# Add origin
# TODO
return repo
@classmethod
def clone_bare(cls, *args, **kwargs):
"""Same as .clone except clones to a bare repository by default
"""
kwargs.setdefault("bare", True)
return cls.clone(*args, **kwargs)
def _commit(self, committer=None, author=None, message=None, files=None, tree=None, *args, **kwargs):
if not tree:
# If no tree then stage files
modified_files = files or self.modified_files
logging.warning("STAGING : %s" % modified_files)
self.add(modified_files)
# Messages
message = message or self.DEFAULT_MESSAGE
author_msg = self._format_userinfo(author)
committer_msg = self._format_userinfo(committer)
return self.repo.do_commit(
message=message, author=author_msg, committer=committer_msg, encoding="UTF-8", tree=tree, *args, **kwargs
)
def _tree_from_structure(self, structure):
# TODO : Support directories
tree = Tree()
for file_info in structure:
# str only
try:
data = file_info["data"].encode("ascii")
name = file_info["name"].encode("ascii")
mode = file_info["mode"]
except:
# Skip file on encoding errors
continue
blob = Blob()
blob.data = data
# Store file's contents
self.repo.object_store.add_object(blob)
# Add blob entry
tree.add(name, mode, blob.id)
# Store tree
self.repo.object_store.add_object(tree)
return tree.id
# Like: git commmit -a
def commit(self, name=None, email=None, message=None, files=None, *args, **kwargs):
user_info = {"name": name, "email": email}
return self._commit(committer=user_info, author=user_info, message=message, files=files, *args, **kwargs)
def commit_structure(self, name=None, email=None, message=None, structure=None, *args, **kwargs):
"""Main use is to do commits directly to bare repositories
For example doing a first Initial Commit so the repo can be cloned and worked on right away
"""
if not structure:
return
tree = self._tree_from_structure(structure)
user_info = {"name": name, "email": email}
return self._commit(committer=user_info, author=user_info, message=message, tree=tree, *args, **kwargs)
# Push all local commits
# and pull all remote commits
def sync(self, origin_uri=None):
self.push(origin_uri)
return self.pull(origin_uri)
def lookup_entry(self, relpath, trackable_files=set()):
if not relpath in trackable_files:
raise KeyError
abspath = self.abspath(relpath)
with open(abspath, "rb") as git_file:
data = git_file.read()
s = sha1()
s.update("blob %u\0" % len(data))
s.update(data)
return (s.hexdigest(), os.stat(abspath).st_mode)
@property
@funky.transform(set)
def tracked_files(self):
return list(self.index)
@property
@funky.transform(set)
def raw_files(self):
return utils.paths.subpaths(self.path)
@property
@funky.transform(set)
def ignored_files(self):
return utils.paths.subpaths(self.path, filters=self.filters)
@property
@funky.transform(set)
def trackable_files(self):
return self.raw_files - self.ignored_files
@property
@funky.transform(set)
def untracked_files(self):
return self.trackable_files - self.tracked_files
"""
@property
@funky.transform(set)
def modified_staged_files(self):
"Checks if the file has changed since last commit"
timestamp = self.last_commit.commit_time
index = self.index
return [
f
for f in self.tracked_files
if index[f][1][0] > timestamp
]
"""
# Return a list of tuples
# representing the changed elements in the git tree
def _changed_entries(self, ref=None):
ref = ref or self.DEFAULT_COMMIT
if not self.has_commits:
return []
obj_sto = self.repo.object_store
tree_id = self[ref].tree
names = self.trackable_files
lookup_func = partial(self.lookup_entry, trackable_files=names)
# Format = [((old_name, new_name), (old_mode, new_mode), (old_sha, new_sha)), ...]
tree_diff = changes_from_tree(names, lookup_func, obj_sto, tree_id, want_unchanged=False)
return list(tree_diff)
@funky.transform(set)
def _changed_entries_by_pattern(self, pattern):
changed_entries = self._changed_entries()
filtered_paths = [
funky.first_true(names)
for names, modes, sha in changed_entries
if tuple(map(bool, names)) == pattern and funky.first_true(names)
]
return filtered_paths
@property
@funky.transform(set)
def removed_files(self):
return self._changed_entries_by_pattern(self.PATTERN_REMOVED) - self.ignored_files
@property
@funky.transform(set)
def added_files(self):
return self._changed_entries_by_pattern(self.PATTERN_ADDED) - self.ignored_files
@property
@funky.transform(set)
def modified_files(self):
modified_files = self._changed_entries_by_pattern(self.PATTERN_MODIFIED) - self.ignored_files
return modified_files
@property
@funky.transform(set)
def modified_unstaged_files(self):
timestamp = self.last_commit.commit_time
return [f for f in self.tracked_files if os.stat(self.abspath(f)).st_mtime > timestamp]
@property
def pending_files(self):
"""
Returns a list of all files that could be possibly staged
"""
# Union of both
return self.modified_files | self.added_files | self.removed_files
@property
def pending_files_by_state(self):
files = {"modified": self.modified_files, "added": self.added_files, "removed": self.removed_files}
# "Flip" the dictionary
return {path: state for state, paths in files.items() for path in paths}
"""
@property
@funky.transform(set)
def modified_files(self):
return self.modified_staged_files | self.modified_unstaged_files
"""
# Like: git add
@funky.arglist_method
def stage(self, files):
return self.repo.stage(files)
def add(self, *args, **kwargs):
return self.stage(*args, **kwargs)
# Like: git rm
@funky.arglist_method
def rm(self, files, force=False):
index = self.index
index_files = filter(lambda f: f in index, files)
for f in index_files:
del self.index[f]
return index.write()
def mv_fs(self, file_pair):
old_name, new_name = file_pair
os.rename(old_name, new_name)
# Like: git mv
@funky.arglist_method
def mv(self, files_pair):
index = self.index
files_in_index = filter(lambda f: f[0] in index, files_pair)
map(self.mv_fs, files_in_index)
old_files = map(funky.first, files_in_index)
new_files = map(funky.last, files_in_index)
self.add(new_files)
self.rm(old_files)
self.add(old_files)
return
@working_only
def _checkout_tree(self, tree):
return build_index_from_tree(self.repo.path, self.repo.index_path(), self.repo.object_store, tree)
def checkout_all(self, commit_sha=None):
commit_sha = commit_sha or self.head
commit_tree = self._commit_tree(commit_sha)
# Rebuild index from the current tree
return self._checkout_tree(commit_tree)
def checkout(self, commit_sha=None, files=None):
"""Checkout only a select amount of files
"""
commit_sha = commit_sha or self.head
files = files or []
return self
@funky.arglist_method
def reset(self, files, commit="HEAD"):
pass
def rm_all(self):
self.index.clear()
return self.index.write()
def _to_commit(self, commit_obj):
"""Allows methods to accept both SHA's or dulwich Commit objects as arguments
"""
if isinstance(commit_obj, basestring):
return self.repo[commit_obj]
return commit_obj
def _commit_sha(self, commit_obj):
"""Extracts a Dulwich commits SHA
"""
if utils.git.is_sha(commit_obj):
return commit_obj
elif isinstance(commit_obj, basestring):
# Can't use self[commit_obj] to avoid infinite recursion
commit_obj = self.repo[commit_obj]
return commit_obj.id
def _blob_data(self, sha):
"""Return a blobs content for a given SHA
"""
return self[sha].data
# Get the nth parent back for a given commit
def get_parent_commit(self, commit, n=None):
""" Recursively gets the nth parent for a given commit
Warning: Remember that parents aren't the previous commits
"""
if n is None:
n = 1
commit = self._to_commit(commit)
parents = commit.parents
if n <= 0 or not parents:
# Return a SHA
return self._commit_sha(commit)
parent_sha = parents[0]
parent = self[parent_sha]
# Recur
return self.get_parent_commit(parent, n - 1)
def get_previous_commit(self, commit_ref, n=None):
commit_sha = self._parse_reference(commit_ref)
n = n or 1
commits = self.commits()
return funky.next(commits, commit_sha, n=n, default=commit_sha)
def _parse_reference(self, ref_string):
# COMMIT_REF~x
if "~" in ref_string:
ref, count = ref_string.split("~")
count = int(count)
commit_sha = self._commit_sha(ref)
return self.get_previous_commit(commit_sha, count)
return self._commit_sha(ref_string)
def _commit_tree(self, commit_sha):
"""Return the tree object for a given commit
"""
return self[commit_sha].tree
def diff(self, commit_sha, compare_to=None, diff_type=None, filter_binary=True):
diff_type = diff_type or self.DEFAULT_DIFF_TYPE
diff_func = self.DIFF_FUNCTIONS[diff_type]
if not compare_to:
compare_to = self.get_previous_commit(commit_sha)
return self._diff_between(compare_to, commit_sha, diff_function=diff_func)
def diff_working(self, ref=None, filter_binary=True):
"""Diff between the current working directory and the HEAD
"""
return utils.git.diff_changes_paths(
self.repo.object_store, self.path, self._changed_entries(ref=ref), filter_binary=filter_binary
)
def get_commit_files(self, commit_sha, parent_path=None, is_tree=None, paths=None):
"""Returns a dict of the following Format :
{
"directory/filename.txt": {
'name': 'filename.txt',
'path': "directory/filename.txt",
"sha": "xxxxxxxxxxxxxxxxxxxx",
"data": "blablabla",
"mode": 0xxxxx",
},
...
}
"""
# Default values
context = {}
is_tree = is_tree or False
parent_path = parent_path or ""
if is_tree:
tree = self[commit_sha]
else:
tree = self[self._commit_tree(commit_sha)]
for mode, path, sha in tree.entries():
# Check if entry is a directory
if mode == self.MODE_DIRECTORY:
context.update(
self.get_commit_files(sha, parent_path=os.path.join(parent_path, path), is_tree=True, paths=paths)
)
continue
subpath = os.path.join(parent_path, path)
# Only add the files we want
if not (paths is None or subpath in paths):
continue
# Add file entry
context[subpath] = {"name": path, "path": subpath, "mode": mode, "sha": sha, "data": self._blob_data(sha)}
return context
def file_versions(self, path):
"""Returns all commits where given file was modified
"""
versions = []
commits_info = self.commit_info()
seen_shas = set()
for commit in commits_info:
try:
files = self.get_commit_files(commit["sha"], paths=[path])
file_path, file_data = files.items()[0]
except IndexError:
continue
file_sha = file_data["sha"]
if file_sha in seen_shas:
continue
else:
seen_shas.add(file_sha)
# Add file info
commit["file"] = file_data
versions.append(file_data)
return versions
def _diff_between(self, old_commit_sha, new_commit_sha, diff_function=None, filter_binary=True):
"""Internal method for getting a diff between two commits
Please use .diff method unless you have very speciic needs
"""
# If commit is first commit (new_commit_sha == old_commit_sha)
# then compare to an empty tree
if new_commit_sha == old_commit_sha:
old_tree = Tree()
else:
old_tree = self._commit_tree(old_commit_sha)
new_tree = self._commit_tree(new_commit_sha)
return diff_function(self.repo.object_store, old_tree, new_tree, filter_binary=filter_binary)
def changes(self, *args, **kwargs):
""" List of changes between two SHAs
Returns a list of lists of tuples :
[
[
(oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
],
...
]
"""
kwargs["diff_type"] = "changes"
return self.diff(*args, **kwargs)
def changes_count(self, *args, **kwargs):
return len(self.changes(*args, **kwargs))
def _refs_by_pattern(self, pattern):
refs = self.refs
def item_filter(key_value):
"""Filter only concered refs"""
key, value = key_value
return key.startswith(pattern)
def item_map(key_value):
"""Rewrite keys"""
key, value = key_value
new_key = key[len(pattern) :]
return (new_key, value)
return dict(map(item_map, filter(item_filter, refs.items())))
@property
def refs(self):
return self.repo.get_refs()
def set_refs(refs_dict):
for k, v in refs_dict.items():
self.repo[k] = v
def import_refs(self, base, other):
return self.repo.refs.import_refs(base, other)
@property
def branches(self):
return self._refs_by_pattern(self.REFS_BRANCHES)
def _active_branch(self, refs=None, head=None):
head = head or self.head
refs = refs or self.branches
try:
return {branch: branch_head for branch, branch_head in refs.items() if branch_head == head}.items()[0]
except IndexError:
pass
return (None, None)
@property
def active_branch(self):
return self._active_branch()[0]
@property
def active_sha(self):
return self._active_branch()[1]
@property
def remote_branches(self):
return self._refs_by_pattern(self.REFS_REMOTES)
@property
def tags(self):
return self._refs_by_pattern(self.REFS_TAGS)
@property
def remotes(self):
""" Dict of remotes
{
'origin': 'http://friendco.de/some_user/repo.git',
...
}
"""
config = self.repo.get_config()
return {keys[1]: values["url"] for keys, values in config.items() if keys[0] == "remote"}
def add_ref(self, new_ref, old_ref):
self.repo.refs[new_ref] = self.repo.refs[old_ref]
self.update_server_info()
def remove_ref(self, ref_name):
# Returns False if ref doesn't exist
if not ref_name in self.repo.refs:
return False
del self.repo.refs[ref_name]
self.update_server_info()
return True
def create_branch(self, base_branch, new_branch, tracking=None):
"""Try creating a new branch which tracks the given remote
if such a branch does not exist then branch off a local branch
"""
# The remote to track
tracking = self.DEFAULT_REMOTE
# Already exists
if new_branch in self.branches:
raise Exception("branch %s already exists" % new_branch)
# Get information about remote_branch
remote_branch = os.path.sep.join([tracking, base_branch])
# Fork Local
if base_branch in self.branches:
base_ref = self._format_ref_branch(base_branch)
# Fork remote
elif remote_branch in self.remote_branches:
base_ref = self._format_ref_remote(remote_branch)
# TODO : track
else:
raise Exception(
"Can not find the branch named '%s' to fork either locally or in '%s'" % (base_branch, tracking)
)
# Reference of new branch
new_ref = self._format_ref_branch(new_branch)
# Copy reference to create branch
self.add_ref(new_ref, base_ref)
return new_ref
def remove_branch(self, branch_name):
ref = self._format_ref_branch(branch_name)
return self.remove_ref(ref)
def switch_branch(self, branch_name, tracking=None, create=None):
"""Changes the current branch
"""
if create is None:
create = True
# Check if branch exists
if not branch_name in self.branches:
self.create_branch(branch_name, branch_name, tracking=tracking)
# Get branch reference
branch_ref = self._format_ref_branch(branch_name)
# Change main branch
self.repo.refs.set_symbolic_ref("HEAD", branch_ref)
if self.is_working:
# Remove all files
self.clean_working()
# Add files for the current branch
self.checkout_all()
def clean(self, force=None, directories=None):
untracked_files = self.untracked_files
map(os.remove, untracked_files)
return untracked_files
def clean_working(self):
"""Purges all the working (removes everything except .git)
used by checkout_all to get clean branch switching
"""
return self.clean()
def _get_fs_structure(self, tree_sha, depth=None, parent_sha=None):
tree = self[tree_sha]
structure = {}
if depth is None:
depth = self.MAX_TREE_DEPTH
elif depth == 0:
return structure
for mode, path, sha in tree.entries():
# tree
if mode == self.MODE_DIRECTORY:
# Recur
structure[path] = self._get_fs_structure(sha, depth=depth - 1, parent_sha=tree_sha)
# commit
else:
structure[path] = sha
structure["."] = tree_sha
structure[".."] = parent_sha or tree_sha
return structure
def _get_fs_structure_by_path(self, tree_sha, path):
parts = path.split(os.path.sep)
depth = len(parts) + 1
structure = self._get_fs_structure(tree_sha, depth=depth)
return funky.subkey(structure, parts)
def commit_ls(self, ref, subpath=None):
"""List a "directory" for a given commit
using the tree of thqt commit
"""
tree_sha = self._commit_tree(ref)
# Root path
if subpath in self.ROOT_PATHS or not subpath:
return self._get_fs_structure(tree_sha, depth=1)
# Any other path
return self._get_fs_structure_by_path(tree_sha, subpath)
def commit_file(self, ref, path):
"""Return info on a given file for a given commit
"""
name, info = self.get_commit_files(ref, paths=[path]).items()[0]
return info
def commit_tree(self, ref, *args, **kwargs):
tree_sha = self._commit_tree(ref)
return self._get_fs_structure(tree_sha, *args, **kwargs)
def update_server_info(self):
if not self.is_bare:
return
update_server_info(self.repo)
def _is_fast_forward(self):
pass
def _merge_fast_forward(self):
pass
def __hash__(self):
"""This is required otherwise the memoize function will just mess it up
"""
return hash(self.path)
def __getitem__(self, key):
sha = self._parse_reference(key)
return self.repo[sha]
def __setitem__(self, key, value):
self.repo[key] = value
# Alias to clone_bare
fork = clone_bare
log = commit_info
diff_count = changes_count
comtributors = recent_contributors
def auth(self, *args, **kwargs):
self.authenticator = GittleAuth(*args, **kwargs)
return self.authenticator
class Gittle(object):
"""All paths used in Gittle external methods must be paths relative to the git repository
"""
DEFAULT_COMMIT = 'HEAD'
DEFAULT_BRANCH = 'master'
DEFAULT_REMOTE = 'origin'
DEFAULT_MESSAGE = '**No Message**'
DEFAULT_USER_INFO = {
'name': None,
'email': None,
}
DIFF_FUNCTIONS = {
'classic': utils.git.classic_tree_diff,
'dict': utils.git.dict_tree_diff,
'changes': utils.git.dict_tree_diff
}
DEFAULT_DIFF_TYPE = 'dict'
HIDDEN_REGEXES = [
# Hide git directory
r'.*\/\.git\/.*',
]
# References
REFS_BRANCHES = 'refs/heads/'
REFS_REMOTES = 'refs/remotes/'
REFS_TAGS = 'refs/tags/'
# Name pattern truths
# Used for detecting if files are :
# - deleted
# - added
# - changed
PATTERN_ADDED = (False, True)
PATTERN_REMOVED = (True, False)
PATTERN_MODIFIED = (True, True)
# Permissions
MODE_DIRECTORY = 0o40000 # Used to tell if a tree entry is a directory
# Tree depth
MAX_TREE_DEPTH = 1000
# Acceptable Root paths
ROOT_PATHS = (os.path.curdir, os.path.sep)
def __init__(self, repo_or_path, origin_uri=None, auth=None, report_activity=None, *args, **kwargs):
if isinstance(repo_or_path, DulwichRepo):
self.repo = repo_or_path
elif isinstance(repo_or_path, Gittle):
self.repo = DulwichRepo(repo_or_path.path)
elif isinstance(repo_or_path, basestring):
path = os.path.abspath(repo_or_path)
self.repo = DulwichRepo(path)
else:
logging.warning('Repo is of type %s' % type(repo_or_path))
raise Exception('Gittle must be initialized with either a dulwich repository or a string to the path')
# Set path
self.path = self.repo.path
# The remote url
self.origin_uri = origin_uri
# Report client activty
self._report_activity = report_activity
# Build ignore filter
self.hidden_regexes = copy.copy(self.HIDDEN_REGEXES)
self.hidden_regexes.extend(self._get_ignore_regexes())
self.ignore_filter = utils.paths.path_filter_regex(self.hidden_regexes)
self.filters = [
self.ignore_filter,
]
# Get authenticator
if auth:
self.authenticator = auth
else:
self.auth(*args, **kwargs)
def report_activity(self, *args, **kwargs):
if not self._report_activity:
return
return self._report_activity(*args, **kwargs)
def _format_author(self, name, email):
return "%s <%s>" % (name, email)
def _format_userinfo(self, userinfo):
name = userinfo.get('name')
email = userinfo.get('email')
if name and email:
return self._format_author(name, email)
return None
def _format_ref(self, base, extra):
return ''.join([base, extra])
def _format_ref_branch(self, branch_name):
return self._format_ref(self.REFS_BRANCHES, branch_name)
def _format_ref_remote(self, remote_name):
return self._format_ref(self.REFS_REMOTES, remote_name)
def _format_ref_tag(self, tag_name):
return self._format_ref(self.REFS_TAGS, tag_name)
@property
def head(self):
"""Return SHA of the current HEAD
"""
return self.repo.head()
@property
def is_bare(self):
"""Bare repositories have no working directories or indexes
"""
return self.repo.bare
@property
def is_working(self):
return not(self.is_bare)
def has_index(self):
"""Opposite of is_bare
"""
return self.repo.has_index()
@property
def has_commits(self):
"""
If the repository has no HEAD we consider that is has no commits
"""
try:
self.repo.head()
except KeyError:
return False
return True
def ref_walker(self, ref=None):
"""
Very simple, basic walker
"""
ref = ref or 'HEAD'
sha = self._commit_sha(ref)
for entry in self.repo.get_walker(sha):
yield entry.commit
def branch_walker(self, branch):
branch = branch or self.active_branch
ref = self._format_ref_branch(branch)
return self.ref_walker(ref)
def commit_info(self, start=0, end=None, branch=None):
"""Return a generator of commits with all their attached information
"""
if not self.has_commits:
return []
commits = [utils.git.commit_info(entry) for entry in self.branch_walker(branch)]
if not end:
return commits
return commits[start:end]
@funky.uniquify
def recent_contributors(self, n=None, branch=None):
n = n or 10
return funky.pluck(self.commit_info(end=n, branch=branch), 'author')
@property
def commit_count(self):
try:
return len(self.ref_walker())
except KeyError:
return 0
def commits(self):
"""Return a list of SHAs for all the concerned commits
"""
return [commit['sha'] for commit in self.commit_info()]
@property
def git_dir(self):
return self.repo.controldir()
def auth(self, *args, **kwargs):
self.authenticator = GittleAuth(*args, **kwargs)
return self.authenticator
# Generate a branch selector (used for pushing)
def _wants_branch(self, branch_name=None):
branch_name = branch_name or self.active_branch
refs_key = self._format_ref_branch(branch_name)
sha = self.branches[branch_name]
def wants_func(old):
refs_key = self._format_ref_branch(branch_name)
return {
refs_key: sha
}
return wants_func
def _get_ignore_regexes(self):
gitignore_filename = os.path.join(self.path, '.gitignore')
if not os.path.exists(gitignore_filename):
return []
lines = open(gitignore_filename).readlines()
globers = map(lambda line: line.rstrip(), lines)
return utils.paths.globers_to_regex(globers)
# Get the absolute path for a file in the git repo
def abspath(self, repo_file):
return os.path.abspath(
os.path.join(self.path, repo_file)
)
# Get the relative path from the absolute path
def relpath(self, abspath):
return os.path.relpath(abspath, self.path)
@property
def last_commit(self):
return self[self.repo.head()]
@property
def index(self):
return self.repo.open_index()
@classmethod
def init(cls, path, bare=None, *args, **kwargs):
"""Initialize a repository"""
mkdir_safe(path)
# Constructor to use
if bare:
constructor = DulwichRepo.init_bare
else:
constructor = DulwichRepo.init
# Create dulwich repo
repo = constructor(path)
# Create Gittle repo
return cls(repo, *args, **kwargs)
@classmethod
def init_bare(cls, *args, **kwargs):
kwargs.setdefault('bare', True)
return cls.init(*args, **kwargs)
@classmethod
def is_repo(cls, path):
"""Returns True if path is a git repository, False if it is not"""
try:
repo = Gittle(path)
except NotGitRepository:
return False
else:
return True
def get_client(self, origin_uri=None, **kwargs):
# Get the remote URL
origin_uri = origin_uri or self.origin_uri
# Fail if inexistant
if not origin_uri:
raise InvalidRemoteUrl()
client_kwargs = {}
auth_kwargs = self.authenticator.kwargs()
client_kwargs.update(auth_kwargs)
client_kwargs.update(kwargs)
client_kwargs.update({
'report_activity': self.report_activity
})
client, remote_path = get_transport_and_path(origin_uri, **client_kwargs)
return client, remote_path
def push_to(self, origin_uri, branch_name=None, progress=None):
selector = self._wants_branch(branch_name=branch_name)
client, remote_path = self.get_client(origin_uri)
return client.send_pack(
remote_path,
selector,
self.repo.object_store.generate_pack_contents,
progress=progress
)
# Like: git push
def push(self, origin_uri=None, branch_name=None, progress=None):
return self.push_to(origin_uri, branch_name, progress)
# Not recommended at ALL ... !!!
def dirty_pull_from(self, origin_uri, branch_name=None):
# Remove all previously existing data
rmtree(self.path)
mkdir_safe(self.path)
self.repo = DulwichRepo.init(self.path)
# Fetch brand new copy from remote
return self.pull_from(origin_uri, branch_name)
def pull_from(self, origin_uri, branch_name=None):
return self.fetch(origin_uri)
# Like: git pull
def pull(self, origin_uri=None, branch_name=None):
return self.pull_from(origin_uri, branch_name)
def fetch_remote(self, origin_uri=None):
# Get client
client, remote_path = self.get_client(origin_uri=origin_uri)
# Fetch data from remote repository
remote_refs = client.fetch(remote_path, self.repo)
return remote_refs
def _setup_fetched_refs(self, refs, origin, bare):
remote_tags = utils.git.subrefs(refs, 'refs/tags')
remote_heads = utils.git.subrefs(refs, 'refs/heads')
# Filter refs
clean_remote_tags = utils.git.clean_refs(remote_tags)
clean_remote_heads = utils.git.clean_refs(remote_heads)
# Base of new refs
heads_base = 'refs/remotes/' + origin
if bare:
heads_base = 'refs/heads'
# Import branches
self.import_refs(
heads_base,
clean_remote_heads
)
# Import tags
self.import_refs(
'refs/tags',
clean_remote_tags
)
# Update HEAD
for k, v in utils.git.clean_refs(refs).items():
self[k] = v
def fetch(self, origin_uri=None, bare=None, origin=None):
bare = bare or False
origin = origin or self.DEFAULT_REMOTE
# Remote refs
remote_refs = self.fetch_remote(origin_uri)
# Update head
# Hit repo because head doesn't yet exist so
# print("REFS = %s" % remote_refs)
# If no refs (empty repository()
if not remote_refs:
return
# Update refs (branches, tags, HEAD)
self._setup_fetched_refs(remote_refs, origin, bare)
# Checkout working directories
if not bare and self.has_commits:
self.checkout_all()
else:
self.update_server_info()
@classmethod
def clone(cls, origin_uri, local_path, auth=None, mkdir=True, bare=False, *args, **kwargs):
"""Clone a remote repository"""
mkdir_safe(local_path)
# Initialize the local repository
if bare:
local_repo = cls.init_bare(local_path)
else:
local_repo = cls.init(local_path)
repo = cls(local_repo, origin_uri=origin_uri, auth=auth, *args, **kwargs)
repo.fetch(bare=bare)
# Add origin
repo.add_remote('origin', origin_uri)
return repo
@classmethod
def clone_bare(cls, *args, **kwargs):
"""Same as .clone except clones to a bare repository by default
"""
kwargs.setdefault('bare', True)
return cls.clone(*args, **kwargs)
def _commit(self, committer=None, author=None, message=None, files=None, tree=None, *args, **kwargs):
if not tree:
# If no tree then stage files
modified_files = files or self.modified_files
logging.info("STAGING : %s" % modified_files)
self.repo.stage(modified_files)
# Messages
message = message or self.DEFAULT_MESSAGE
author_msg = self._format_userinfo(author)
committer_msg = self._format_userinfo(committer)
return self.repo.do_commit(
message=message,
author=author_msg,
committer=committer_msg,
encoding='UTF-8',
tree=tree,
*args, **kwargs
)
def _tree_from_structure(self, structure):
# TODO : Support directories
tree = Tree()
for file_info in structure:
# str only
try:
data = file_info['data'].encode('ascii')
name = file_info['name'].encode('ascii')
mode = file_info['mode']
except:
# Skip file on encoding errors
continue
blob = Blob()
blob.data = data
# Store file's contents
self.repo.object_store.add_object(blob)
# Add blob entry
tree.add(
name,
mode,
blob.id
)
# Store tree
self.repo.object_store.add_object(tree)
return tree.id
# Like: git commmit -a
def commit(self, name=None, email=None, message=None, files=None, *args, **kwargs):
user_info = {
'name': name,
'email': email,
}
return self._commit(
committer=user_info,
author=user_info,
message=message,
files=files,
*args,
**kwargs
)
def commit_structure(self, name=None, email=None, message=None, structure=None, *args, **kwargs):
"""Main use is to do commits directly to bare repositories
For example doing a first Initial Commit so the repo can be cloned and worked on right away
"""
if not structure:
return
tree = self._tree_from_structure(structure)
user_info = {
'name': name,
'email': email,
}
return self._commit(
committer=user_info,
author=user_info,
message=message,
tree=tree,
*args,
**kwargs
)
# Push all local commits
# and pull all remote commits
def sync(self, origin_uri=None):
self.push(origin_uri)
return self.pull(origin_uri)
def lookup_entry(self, relpath, trackable_files=set()):
if not relpath in trackable_files:
raise KeyError
abspath = self.abspath(relpath)
with open(abspath, 'rb') as git_file:
data = git_file.read()
s = sha1()
s.update("blob %u\0" % len(data))
s.update(data)
return (s.hexdigest(), os.stat(abspath).st_mode)
@property
@funky.transform(set)
def tracked_files(self):
return list(self.index)
@property
@funky.transform(set)
def raw_files(self):
return utils.paths.subpaths(self.path)
@property
@funky.transform(set)
def ignored_files(self):
return utils.paths.subpaths(self.path, filters=self.filters)
@property
@funky.transform(set)
def trackable_files(self):
return self.raw_files - self.ignored_files
@property
@funky.transform(set)
def untracked_files(self):
return self.trackable_files - self.tracked_files
"""
@property
@funky.transform(set)
def modified_staged_files(self):
"Checks if the file has changed since last commit"
timestamp = self.last_commit.commit_time
index = self.index
return [
f
for f in self.tracked_files
if index[f][1][0] > timestamp
]
"""
# Return a list of tuples
# representing the changed elements in the git tree
def _changed_entries(self, ref=None):
ref = ref or self.DEFAULT_COMMIT
if not self.has_commits:
return []
obj_sto = self.repo.object_store
tree_id = self[ref].tree
names = self.trackable_files
lookup_func = partial(self.lookup_entry, trackable_files=names)
# Format = [((old_name, new_name), (old_mode, new_mode), (old_sha, new_sha)), ...]
tree_diff = changes_from_tree(names, lookup_func, obj_sto, tree_id, want_unchanged=False)
return list(tree_diff)
@funky.transform(set)
def _changed_entries_by_pattern(self, pattern):
changed_entries = self._changed_entries()
filtered_paths = None
#if the pattern is PATTERN_MODIFIED, should check the sha
if self.PATTERN_MODIFIED == pattern:
filtered_paths = [
funky.first_true(names)
for names, modes, sha in changed_entries
if tuple(map(bool, names)) == pattern and funky.first_true(names) and sha[0] == sha[1]
]
else :
filtered_paths = [
funky.first_true(names)
for names, modes, sha in changed_entries
if tuple(map(bool, names)) == pattern and funky.first_true(names)
]
return filtered_paths
@property
@funky.transform(set)
def removed_files(self):
return self._changed_entries_by_pattern(self.PATTERN_REMOVED) - self.ignored_files
@property
@funky.transform(set)
def added_files(self):
return self._changed_entries_by_pattern(self.PATTERN_ADDED) - self.ignored_files
@property
@funky.transform(set)
def modified_files(self):
modified_files = self._changed_entries_by_pattern(self.PATTERN_MODIFIED) - self.ignored_files
return modified_files
@property
@funky.transform(set)
def modified_unstaged_files(self):
timestamp = self.last_commit.commit_time
return [
f
for f in self.tracked_files
if os.stat(self.abspath(f)).st_mtime > timestamp
]
@property
def pending_files(self):
"""
Returns a list of all files that could be possibly staged
"""
# Union of both
return self.modified_files | self.added_files | self.removed_files
@property
def pending_files_by_state(self):
files = {
'modified': self.modified_files,
'added': self.added_files,
'removed': self.removed_files
}
# "Flip" the dictionary
return {
path: state
for state, paths in files.items()
for path in paths
}
"""
@property
@funky.transform(set)
def modified_files(self):
return self.modified_staged_files | self.modified_unstaged_files
"""
# Like: git add
@funky.arglist_method
def stage(self, files):
return self.repo.stage(files)
def add(self, *args, **kwargs):
return self.stage(*args, **kwargs)
# Like: git rm
@funky.arglist_method
def rm(self, files, force=False):
index = self.index
index_files = filter(lambda f: f in index, files)
for f in index_files:
del self.index[f]
return index.write()
def mv_fs(self, file_pair):
old_name, new_name = file_pair
os.rename(old_name, new_name)
# Like: git mv
@funky.arglist_method
def mv(self, files_pair):
index = self.index
files_in_index = filter(lambda f: f[0] in index, files_pair)
map(self.mv_fs, files_in_index)
old_files = map(funky.first, files_in_index)
new_files = map(funky.last, files_in_index)
self.add(new_files)
self.rm(old_files)
self.add(old_files)
return
@working_only
def _checkout_tree(self, tree):
return build_index_from_tree(
self.repo.path,
self.repo.index_path(),
self.repo.object_store,
tree
)
def checkout_all(self, commit_sha=None):
commit_sha = commit_sha or self.head
commit_tree = self._commit_tree(commit_sha)
# Rebuild index from the current tree
return self._checkout_tree(commit_tree)
def checkout(self, ref):
"""Checkout a given ref or SHA
"""
self.repo.refs.set_symbolic_ref('HEAD', ref)
commit_tree = self._commit_tree(ref)
# Rebuild index from the current tree
return self._checkout_tree(commit_tree)
@funky.arglist_method
def reset(self, files, commit='HEAD'):
pass
def rm_all(self):
# if we go at the index via the property, it is reconstructed
# each time and therefore clear() doesn't have the desired effect,
# therefore, we cache it in a variable and use that.
i = self.index
i.clear()
return i.write()
def _to_commit(self, commit_obj):
"""Allows methods to accept both SHA's or dulwich Commit objects as arguments
"""
if isinstance(commit_obj, basestring):
return self.repo[commit_obj]
return commit_obj
def _commit_sha(self, commit_obj):
"""Extracts a Dulwich commits SHA
"""
if utils.git.is_sha(commit_obj):
return commit_obj
elif isinstance(commit_obj, basestring):
# Can't use self[commit_obj] to avoid infinite recursion
commit_obj = self.repo[self.dwim_reference(commit_obj)]
return commit_obj.id
def dwim_reference(self, ref):
"""Dwim resolves a short reference to a full reference
"""
# Formats of refs we want to try in order
formats = [
"%s",
"refs/%s",
"refs/tags/%s",
"refs/heads/%s",
"refs/remotes/%s",
"refs/remotes/%s/HEAD",
]
for f in formats:
try:
fullref = f % ref
if not fullref in self.repo:
continue
return fullref
except:
continue
raise Exception("Could not resolve ref")
def blob_data(self, sha):
"""Return a blobs content for a given SHA
"""
return self[sha].data
# Get the nth parent back for a given commit
def get_parent_commit(self, commit, n=None):
""" Recursively gets the nth parent for a given commit
Warning: Remember that parents aren't the previous commits
"""
if n is None:
n = 1
commit = self._to_commit(commit)
parents = commit.parents
if n <= 0 or not parents:
# Return a SHA
return self._commit_sha(commit)
parent_sha = parents[0]
parent = self[parent_sha]
# Recur
return self.get_parent_commit(parent, n - 1)
def get_previous_commit(self, commit_ref, n=None):
commit_sha = self._parse_reference(commit_ref)
n = n or 1
commits = self.commits()
return funky.next(commits, commit_sha, n=n, default=commit_sha)
def _parse_reference(self, ref_string):
# COMMIT_REF~x
if '~' in ref_string:
ref, count = ref_string.split('~')
count = int(count)
commit_sha = self._commit_sha(ref)
return self.get_previous_commit(commit_sha, count)
return self._commit_sha(ref_string)
def _commit_tree(self, commit_sha):
"""Return the tree object for a given commit
"""
return self[commit_sha].tree
def diff(self, commit_sha, compare_to=None, diff_type=None, filter_binary=True):
diff_type = diff_type or self.DEFAULT_DIFF_TYPE
diff_func = self.DIFF_FUNCTIONS[diff_type]
if not compare_to:
compare_to = self.get_previous_commit(commit_sha)
return self._diff_between(compare_to, commit_sha, diff_function=diff_func)
def diff_working(self, ref=None, filter_binary=True):
"""Diff between the current working directory and the HEAD
"""
return utils.git.diff_changes_paths(
self.repo.object_store,
self.path,
self._changed_entries(ref=ref),
filter_binary=filter_binary
)
def get_commit_files(self, commit_sha, parent_path=None, is_tree=None, paths=None):
"""Returns a dict of the following Format :
{
"directory/filename.txt": {
'name': 'filename.txt',
'path': "directory/filename.txt",
"sha": "xxxxxxxxxxxxxxxxxxxx",
"data": "blablabla",
"mode": 0xxxxx",
},
...
}
"""
# Default values
context = {}
is_tree = is_tree or False
parent_path = parent_path or ''
if is_tree:
tree = self[commit_sha]
else:
tree = self[self._commit_tree(commit_sha)]
for entry in tree.items():
# Check if entry is a directory
if entry.mode == self.MODE_DIRECTORY:
context.update(
self.get_commit_files(entry.sha, parent_path=os.path.join(parent_path, entry.path), is_tree=True, paths=paths)
)
continue
subpath = os.path.join(parent_path, entry.path)
# Only add the files we want
if not(paths is None or subpath in paths):
continue
# Add file entry
context[subpath] = {
'name': entry.path,
'path': subpath,
'mode': entry.mode,
'sha': entry.sha,
'data': self.blob_data(entry.sha),
}
return context
def file_versions(self, path):
"""Returns all commits where given file was modified
"""
versions = []
commits_info = self.commit_info()
seen_shas = set()
for commit in commits_info:
try:
files = self.get_commit_files(commit['sha'], paths=[path])
file_path, file_data = files.items()[0]
except IndexError:
continue
file_sha = file_data['sha']
if file_sha in seen_shas:
continue
else:
seen_shas.add(file_sha)
# Add file info
commit['file'] = file_data
versions.append(file_data)
return versions
def _diff_between(self, old_commit_sha, new_commit_sha, diff_function=None, filter_binary=True):
"""Internal method for getting a diff between two commits
Please use .diff method unless you have very specific needs
"""
# If commit is first commit (new_commit_sha == old_commit_sha)
# then compare to an empty tree
if new_commit_sha == old_commit_sha:
old_tree = Tree()
else:
old_tree = self._commit_tree(old_commit_sha)
new_tree = self._commit_tree(new_commit_sha)
return diff_function(self.repo.object_store, old_tree, new_tree, filter_binary=filter_binary)
def changes(self, *args, **kwargs):
""" List of changes between two SHAs
Returns a list of lists of tuples :
[
[
(oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
],
...
]
"""
kwargs['diff_type'] = 'changes'
return self.diff(*args, **kwargs)
def changes_count(self, *args, **kwargs):
return len(self.changes(*args, **kwargs))
def _refs_by_pattern(self, pattern):
refs = self.refs
def item_filter(key_value):
"""Filter only concered refs"""
key, value = key_value
return key.startswith(pattern)
def item_map(key_value):
"""Rewrite keys"""
key, value = key_value
new_key = key[len(pattern):]
return (new_key, value)
return dict(
map(item_map,
filter(
item_filter,
refs.items()
)
)
)
@property
def refs(self):
return self.repo.get_refs()
def set_refs(refs_dict):
for k, v in refs_dict.items():
self.repo[k] = v
def import_refs(self, base, other):
return self.repo.refs.import_refs(base, other)
@property
def branches(self):
return self._refs_by_pattern(self.REFS_BRANCHES)
@property
def active_branch(self):
"""Returns the name of the active branch, or None, if HEAD is detached
"""
x = self.repo.refs.read_ref('HEAD')
if not x.startswith(SYMREF):
return None
else:
symref = x[len(SYMREF):]
if not symref.startswith(self.REFS_BRANCHES):
return None
else:
return symref[len(self.REFS_BRANCHES):]
@property
def active_sha(self):
"""Deprecated equivalent to head property
"""
return self.head
@property
def remote_branches(self):
return self._refs_by_pattern(self.REFS_REMOTES)
@property
def tags(self):
return self._refs_by_pattern(self.REFS_TAGS)
@property
def remotes(self):
""" Dict of remotes
{
'origin': 'http://friendco.de/some_user/repo.git',
...
}
"""
config = self.repo.get_config()
return {
keys[1]: values['url']
for keys, values in config.items()
if keys[0] == 'remote'
}
def add_remote(self, remote_name, remote_url):
# Get repo's config
config = self.repo.get_config()
# Add new entries for remote
config.set(('remote', remote_name), 'url', remote_url)
config.set(('remote', remote_name), 'fetch', "+refs/heads/*:refs/remotes/%s/*" % remote_name)
# Write to disk
config.write_to_path()
return remote_name
def add_ref(self, new_ref, old_ref):
self.repo.refs[new_ref] = old_ref
self.update_server_info()
def remove_ref(self, ref_name):
# Returns False if ref doesn't exist
if not ref_name in self.repo.refs:
return False
del self.repo.refs[ref_name]
self.update_server_info()
return True
def create_branch(self, base_branch, new_branch, tracking=None):
"""Try creating a new branch which tracks the given remote
if such a branch does not exist then branch off a local branch
"""
# The remote to track
tracking = self.DEFAULT_REMOTE
# Already exists
if new_branch in self.branches:
raise Exception("branch %s already exists" % new_branch)
# Get information about remote_branch
remote_branch = os.path.sep.join([tracking, base_branch])
# Fork Local
if base_branch in self.branches:
base_ref = self._format_ref_branch(base_branch)
# Fork remote
elif remote_branch in self.remote_branches:
base_ref = self._format_ref_remote(remote_branch)
# TODO : track
else:
raise Exception("Can not find the branch named '%s' to fork either locally or in '%s'" % (base_branch, tracking))
# Reference of new branch
new_ref = self._format_ref_branch(new_branch)
# Copy reference to create branch
self.add_ref(new_ref, base_ref)
return new_ref
def create_orphan_branch(self, new_branch, empty_index=None):
""" Create a new branch with no commits in it.
Technically, just points HEAD to a non-existent branch. The actual branch will
only be created if something is committed. This is equivalent to:
git checkout --orphan <new_branch>,
Unless empty_index is set to True, in which case the index will be emptied along
with the file-tree (which is always emptied). Against a clean working tree,
this is equivalent to:
git checkout --orphan <new_branch>
git reset --merge
"""
if new_branch in self.branches:
raise Exception("branch %s already exists" % new_branch)
new_ref = self._format_ref_branch(new_branch)
self.repo.refs.set_symbolic_ref('HEAD', new_ref)
if self.is_working:
if empty_index:
self.rm_all()
self.clean_working()
return new_ref
def remove_branch(self, branch_name):
ref = self._format_ref_branch(branch_name)
return self.remove_ref(ref)
def switch_branch(self, branch_name, tracking=None, create=None):
"""Changes the current branch
"""
if create is None:
create = True
# Check if branch exists
if not branch_name in self.branches:
self.create_branch(branch_name, branch_name, tracking=tracking)
# Get branch reference
branch_ref = self._format_ref_branch(branch_name)
# Change main branch
self.repo.refs.set_symbolic_ref('HEAD', branch_ref)
if self.is_working:
# Remove all files
self.clean_working()
# Add files for the current branch
self.checkout_all()
def create_tag(self, tag_name, target):
ref = self._format_ref_tag(tag_name)
return self.add_ref(ref, self._parse_reference(target))
def remove_tag(self, tag_name):
ref = self._format_ref_tag(tag_name)
return self.remove_ref(ref)
def clean(self, force=None, directories=None):
untracked_files = self.untracked_files
map(os.remove, untracked_files)
return untracked_files
def clean_working(self):
"""Purges all the working (removes everything except .git)
used by checkout_all to get clean branch switching
"""
return self.clean()
def _get_fs_structure(self, tree_sha, depth=None, parent_sha=None):
tree = self[tree_sha]
structure = {}
if depth is None:
depth = self.MAX_TREE_DEPTH
elif depth == 0:
return structure
for entry in tree.items():
# tree
if entry.mode == self.MODE_DIRECTORY:
# Recur
structure[entry.path] = self._get_fs_structure(entry.sha, depth=depth - 1, parent_sha=tree_sha)
# commit
else:
structure[entry.path] = entry.sha
structure['.'] = tree_sha
structure['..'] = parent_sha or tree_sha
return structure
def _get_fs_structure_by_path(self, tree_sha, path):
parts = path.split(os.path.sep)
depth = len(parts) + 1
structure = self._get_fs_structure(tree_sha, depth=depth)
return funky.subkey(structure, parts)
def commit_ls(self, ref, subpath=None):
"""List a "directory" for a given commit
using the tree of that commit
"""
tree_sha = self._commit_tree(ref)
# Root path
if subpath in self.ROOT_PATHS or not subpath:
return self._get_fs_structure(tree_sha, depth=1)
# Any other path
return self._get_fs_structure_by_path(tree_sha, subpath)
def commit_file(self, ref, path):
"""Return info on a given file for a given commit
"""
name, info = self.get_commit_files(ref, paths=[path]).items()[0]
return info
def commit_tree(self, ref, *args, **kwargs):
tree_sha = self._commit_tree(ref)
return self._get_fs_structure(tree_sha, *args, **kwargs)
def update_server_info(self):
if not self.is_bare:
return
update_server_info(self.repo)
def _is_fast_forward(self):
pass
def _merge_fast_forward(self):
pass
def __hash__(self):
"""This is required otherwise the memoize function will just mess it up
"""
return hash(self.path)
def __getitem__(self, key):
try:
sha = self._parse_reference(key)
except:
raise KeyError(key)
return self.repo[sha]
def __setitem__(self, key, value):
try:
key = self.dwim_reference(key)
except:
pass
self.repo[key] = value
def __contains__(self, key):
try:
key = self.dwim_reference(key)
except:
pass
return key in self.repo
def __delitem__(self, key):
try:
key = self.dwim_reference(key)
except:
raise KeyError(key)
self.remove_ref(key)
# Alias to clone_bare
fork = clone_bare
log = commit_info
diff_count = changes_count
contributors = recent_contributors
def auth(self, *args, **kwargs):
self.authenticator = GittleAuth(*args, **kwargs)
return self.authenticator
def post(self):
#authuserList=['jino']
#if not self.get_current_user() in authuserList:
# self.write('unauthorized users')
# return
parser = CappConfig(CONF)
self.set_header('Content-Type', 'text/plain')
#sid=self.request.arguments['sid']
cid = self.get_argument('cid', None)
if not cid:
self.write('ERROR:invalid branch')
return
sid = self.get_argument('sid', None)
if not sid:
self.write('ERROR:invalid server')
return
dst = get_srvlist(sid)
servname = parser.get('REPO', '%s.servname' % dst)
host = parser.get('SERVER', '%s.hostname' % servname)
user = parser.get('SERVER', '%s.username' % servname)
pwds = parser.get('SERVER', '%s.password' % servname)
keyd = parser.get('PATH', 'keysdire')
pkey = os.path.join(keyd, parser.get('SERVER',
'%s.pkeyfile' % servname))
if not os.path.exists(pkey):
self.write('ERROR:pkeyfile not exists or chmod 600 %s user %s' %
(pkey, getpass.getuser()))
return
if int(oct(os.stat(pkey).st_mode)[-3:]) != 600:
self.write('MESSAGE:pls chmod 600 %s user %s' %
(pkey, getpass.getuser()))
return
bpre = parser.get('SERVER', '%s.branchpx' % servname)
port = int(parser.get('SERVER', '%s.port' % servname))
sdir = parser.get('REPO', '%s.syncdirs' % dst)
item = parser.get('REPO', '%s.projitem' % dst)
cper = parser.get('REPO', '%s.permissi' % dst)
if not re.match(bpre.replace(',', '|'), cid):
self.write('ERROR:branch not right')
return
#for b in bpre.split(','):
# if not re.match(b,cid):
# self.write('ERROR:branch not right')
# return
workdirs = parser.get('PATH', 'workdirs')
verconfile = os.path.join(workdirs, 'vercnf')
verconf = CappConfig(verconfile)
try:
vers = verconf.get('REPO', '%s.versions' % dst)
except:
vers = ''
dskey = os.path.join(keyd, parser.get('PATH', 'deploykf'))
if not os.path.exists(dskey):
self.write('ERROR:deploykf not exists or chmod 600 %s user %s' %
(dskey, getpass.getuser()))
return
if int(oct(os.stat(dskey).st_mode)[-3:]) != 600:
self.write('MESSAGE:pls chmod 600 %s user %s' %
(dskey, getpass.getuser()))
return
try:
authk = GittleAuth(pkey=dskey)
except:
self.write("ERROR:deploy key valid")
return
try:
repo = Gittle(os.path.join(workdirs, dst),
origin_uri=repo_url[item],
auth=authk)
except:
try:
repo = Gittle.clone(repo_url[item],
os.path.join(workdirs, dst),
auth=authk)
except:
self.write(
'ERROR:deploy key not permission or confdirs cannot write')
return
try:
repo.switch_branch('master')
repo.pull_from(repo_url[item], cid)
repo.switch_branch(cid)
lastcommit = repo.commit_info(0, 1, cid)[0]
self.write('%s\t%s\n%s\n' %
(lastcommit['summary'][:60], lastcommit['sha'][:8],
lastcommit['committer']['name']))
except:
try:
rmtree(os.path.join(workdirs, dst))
repo = Gittle.clone(repo_url[item],
os.path.join(workdirs, dst),
auth=authk)
repo.switch_branch('master')
repo.pull_from(repo_url[item], cid)
repo.switch_branch(cid)
lastcommit = repo.commit_info(0, 1, cid)[0]
self.write('%s\t%s\n%s\n' %
(lastcommit['summary'][:60], lastcommit['sha'][:8],
lastcommit['committer']['name']))
except Exception as e:
self.write('confdirs cannot write')
return
parser2 = CappConfig(parser.get('PATH', 'excludef'))
try:
parser2.items('%s%s%s' % (servname, item, dst))
except:
self.write('ERROR:excludef no section %s%s%s' %
(servname, item, dst))
return
self.write('SETP1 \tstart:\n')
for sfile, dfile in parser2.items('%s%s%s' % (servname, item, dst)):
ssdir = os.path.join(parser.get('PATH', 'confdirs'),
'%s%s%s' % (servname, item, dst))
ddir = os.path.join(workdirs, dst)
if dfile.startswith('/'): dfile = dfile[1:]
err, out = rsync(os.path.join(ssdir, sfile),
os.path.join(ddir, dfile))
if len(err) is 0:
reChar = re.compile(
'send*.*list|send*.*sec|total size*.*|rsync:')
self.write('%s' % reChar.sub('', out))
else:
reChar = re.compile('rsync error:*.*|rsync: link_stat|rsync:')
self.write('ERROR:%s' % reChar.sub('', err))
return
self.write('SETP1 \tend.\n')
conflist = os.path.join(ddir, '.git/conflist')
if os.path.exists(conflist):
rsync_arg = '--exclude=.git --exclude-from=%s' % os.path.join(
ddir, '.git/conflist')
else:
rsync_arg = '--exclude=.git'
local_dir = '%s/' % ddir
remote_dir = '%s@%s:%s' % (user, host, sdir)
err, _ = sshto(host, int(port), user, pwds, pkey).exe_cmdOne('')
if err:
self.write('ERROR:target server auth failure')
return
only_com = False
try:
precmd, aftcmd = parser.get('REPO', '%s.cmdexecu' % dst).split(',')
except:
precmd = parser.get('REPO', '%s.cmdexecu' % dst)
aftcmd = ''
if os.path.isfile(precmd): precmd = open(precmd).read()
if os.path.isfile(aftcmd): aftcmd = open(aftcmd).read()
if self.get_argument('check', None) is None:
if not self.get_current_user()['username'] + '|c' in cper:
if precmd:
precmd_err, precmd_out = sshto(host, int(port), user, pwds,
pkey).exe_cmdOne(precmd)
if len(precmd_err) is 0:
self.write('EXEC PRE:\n%s\n' % precmd_out)
else:
self.write('ERROR: exec pre\n%s\n' % precmd_err)
return
err2, out2 = rsync(local_dir,
remote_dir,
arg=rsync_arg,
pkey=pkey)
if aftcmd:
aftcmd_err, aftcmd_out = sshto(host, int(port), user, pwds,
pkey).exe_cmdOne(aftcmd)
if len(aftcmd_err) is 0:
self.write('EXEC AFT:\n%s\n' % aftcmd_out)
else:
self.write('ERROR: exec aft\n%s\n' % aftcmd_err)
if out2:
config = ConfigParser.RawConfigParser()
if vers:
vers = vers.split(',')
if len(vers) == 1: vers.append(cid)
vers[1] = vers[0]
vers[0] = cid
config.read(verconfile)
config.set('REPO', '%s.versions' % dst, ','.join(vers))
with open(verconfile, 'w') as configfile:
config.write(configfile)
else:
config.read(verconfile)
config.set('REPO', '%s.versions' % dst, cid)
config.write(open(verconfile, 'w'))
else:
self.write('ERROR:no publishing rights\n')
return
else:
self.write('EXEC PRE:\n%s\n' % precmd)
err2, out2 = rsync(local_dir,
remote_dir,
arg=rsync_arg,
pkey=pkey,
check=True)
self.write('EXEC AFT:\n%s\n' % aftcmd)
only_com = True
if len(err2) is 0:
reChar = re.compile('send*.*list|send*.*sec|total size*.*|rsync:')
self.write('SETP2 \tstart:\n')
self.write('%s' % reChar.sub('', out2))
self.write('SETP2 \tend.\n')
else:
reChar = re.compile('rsync error:*.*|rsync: link_stat|rsync:')
self.write('%s' % reChar.sub('', err2))
if only_com:
self.write('MESSAGE:only for comparison is not pub\n')
return
