def git_is_clean(srcdir, project): repo = Repository(os.path.join(srcdir, project.workspace_path, ".git")) for _, b in iteritems(repo.status()): if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT: return False, "has uncommitted changes" if repo.head_is_detached: return False, "has detached HEAD" origin = get_origin(repo, project) if not origin: return False, "has no upstream remote" remote_refs = [] local_refs = {} for refname in repo.listall_references(): if refname.startswith("refs/remotes/%s/" % origin.name): ref = repo.lookup_reference(refname) if ref.type == GIT_REF_OID: remote_refs.append(ref.target) elif not refname.startswith("refs/remotes/"): ref = repo.lookup_reference(refname) if ref.type == GIT_REF_OID: local_refs[ref.peel().id] = refname if not remote_refs: return False, "has no upstream remote branches" if not local_refs: return False, "has no local branches" if not repo.lookup_branch("%s/%s" % (origin.name, project.master_branch), GIT_BRANCH_REMOTE): return False, "has no upstream master branch" for remote_ref in remote_refs: for commit in repo.walk(remote_ref): if commit.id in local_refs: del local_refs[commit.id] if local_refs: return False, "has local commits: %s" % ", ".join(["'%s'" % name for _, name in iteritems(local_refs)]) return True, ""
def get_docset_props(docset: str, config: Dict, manifest: Manifest) -> Tuple[str, bool]: """Obtain the id and dirty status of the given docset. Args: docset: Docset name. config: Cache configuration. manifest: Repository manifest. Returns: Docset ID. """ for entry in config: if entry["docset"] != docset: continue id = "" dirty = False for name in entry["projects"]: p = next((p for p in manifest.projects if p.name == name), None) assert p, f"Project {name} not in manifest" repo = Repository(Path(p.topdir) / p.path) id += repo.revparse_single("HEAD").id.hex dirty = dirty or bool(repo.status()) return hashlib.sha256(id.encode("utf-8")).hexdigest(), dirty raise ValueError(f"Docset {docset} not in configuration file")
def is_clean(repo: pygit2.Repository) -> bool: return not any( code for code in repo.status().values() if ( code ^ (code & pygit2.GIT_STATUS_WT_NEW) ^ (code & pygit2.GIT_STATUS_IGNORED) ) )
def get_deleted_files(path_to_repository): """Utility method for getting the deleted files from a Git repository. Args: path_to_repository (str): Path to the Git repository Returns: List(str): List of filenames of all deleted files in the provided repository. """ repo = Repository(path_to_repository) status_entries = repo.status() return [path for path, st in status_entries.items() if st & _STATI_DELETED]
def get_changed_files(path_to_repository): """Utility method for getting the currently changed files from a Git repository. Args: path_to_repository (str): Path to the Git repository Returns: List(str): List of filenames of all changed files in the provided repository. """ repo = Repository(path_to_repository) status_entries = repo.status() return [path for path, st in status_entries.items() if st & _STATI_CONSIDERED_FOR_PRECOMMIT]
def is_repo_dirty(repo: pygit2.Repository) -> bool: """Check if a repository is dirty (not clean). Args: repo: Repository. Returns: True if dirty, False otherwise. """ return bool([ f for f, code in repo.status().items() if code != pygit2.GIT_STATUS_IGNORED ])
def update_repo(reponame): """ For a given path to a repo, pull/rebase the last changes if it can, add/remove/commit the new changes and push them to the remote repo if any. :kwarg reponame, full path to a git repo. """ LOG.info('Processing %s' % reponame) if not os.path.exists(reponame): raise GitSyncError( 'The indicated working directory does not exists: %s' % reponame) try: repo = Repository(reponame) except Exception as err: print(err) raise GitSyncError( 'The indicated working directory is not a valid git ' 'repository: %s' % reponame) index = repo.index dopush = False origin = None index = repo.index ## Add or remove to staging the files according to their status if repo.status: status = repo.status() for filepath, flag in status.items(): if flag == GIT_STATUS_WT_DELETED: msg = 'Remove file %s' % filepath LOG.info(msg) index.remove(filepath) docommit(repo, index, msg) dopush = True elif flag == GIT_STATUS_WT_NEW: msg = 'Add file %s' % filepath LOG.info(msg) index.add(filepath) docommit(repo, index, msg) dopush = True elif flag == GIT_STATUS_WT_MODIFIED: msg = 'Change file %s' % filepath LOG.info(msg) index.add(filepath) docommit(repo, index, msg) dopush = True return dopush
def commit_changes(file: str, repo_path: Path): """Commit changes on file to the repository at repo_path.""" repo = Repository(repo_path) for path, flags in repo.status().items(): if path == file and (flags != GIT_STATUS_WT_MODIFIED and flags != GIT_STATUS_WT_NEW): return print(f'commiting {file}') repo.index.add(file) repo.index.write() tree = repo.index.write_tree() old_head = repo.head.peel(Commit).id repo.create_commit('refs/heads/master', SIGNATURE, SIGNATURE, f'[build-server]: update {file}', tree, [old_head])
def changed_includes(conf): from pygit2 import Repository, GIT_STATUS_CURRENT, GIT_STATUS_IGNORED repo_path = conf.paths.projectroot r = Repository(repo_path) changed = [] for path, flag in r.status().items(): if flag not in [GIT_STATUS_CURRENT, GIT_STATUS_IGNORED]: if path.startswith('source/'): if path.endswith('.txt') or path.endswith('.rst'): changed.append(path[6:]) changed_report = [] for fn in include_files(conf): if fn in changed: changed_report.append(fn) return changed_report
def detect_changed_files(repo: pygit2.Repository, repo_path: Path) -> Iterator[Path]: submodules = repo.listall_submodules() for file, flags in repo.status().items(): if flags not in (pygit2.GIT_STATUS_CURRENT, pygit2.GIT_STATUS_IGNORED): target_path = Path(repo_path, file) if not target_path.is_dir(): yield target_path else: relative_path = target_path.relative_to(repo_path) # NOTE: Special treatment for sub-modules if str(relative_path) in submodules: sub_repo = pygit2.Repository(target_path) # TODO: What if it's no longer a repository? # Mark the subrepo itself as modified. It has additional commit metadata # that might have changed yield target_path # Detect any modified files within the sub-repo # NOTE: This is faster than plain hashing because it implicitly takes advantage of # the tracking git has already done. yield from detect_changed_files(sub_repo, target_path) else: # Not a submodule: just mark all (non-ignored) subfiles as changed # # NOTE: We do not yield the directory itself because git ignores that. # There is no extra metadata to add in that case. detected_modification = False for dirpath, dirnames, filenames in os.walk(target_path): relative_dirpath = Path(dirpath).relative_to(repo_path) for name in filenames: if not repo.path_is_ignored(str(Path(relative_dirpath, name))): yield Path(dirpath, name) detected_modification = True for sub_dir in list(dirnames): if repo.path_is_ignored(str(Path(relative_dirpath, sub_dir))): dirnames.remove(sub_dir) else: yield Path(dirpath, sub_dir) detected_modification = True if not detected_modification: raise AssertionError(f"Unable to find git's claimed modification (flags={flags:04x}): {target_path}")
def changed_includes(conf=None): from pygit2 import Repository, GIT_STATUS_CURRENT, GIT_STATUS_IGNORED conf = lazy_conf(conf) repo_path = conf.paths.projectroot r = Repository(repo_path) changed = [] for path, flag in r.status().items(): if flag not in [ GIT_STATUS_CURRENT, GIT_STATUS_IGNORED ]: if path.startswith('source/'): if path.endswith('.txt'): changed.append(path[6:]) source_path = os.path.join(conf.paths.source, conf.paths.output, conf.git.branches.current, 'json') changed_report = [] for report in _generate_report(None): if report['source'][len(source_path):] in changed: changed_report.append(report) return changed_report
def git_is_clean(srcdir, project): repo = Repository(os.path.join(srcdir, project.workspace_path, ".git")) for _, b in iteritems(repo.status()): if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT: return False, "has uncommitted changes" if repo.head_is_detached: return False, "has detached HEAD" origin = get_origin(repo, project) if not origin: return False, "has no upstream remote" remote_refs = [] local_refs = {} for refname in repo.listall_references(): if refname.startswith("refs/remotes/%s/" % origin.name): ref = repo.lookup_reference(refname) if ref.type == GIT_REF_OID: remote_refs.append(ref.target) elif not refname.startswith("refs/remotes/"): ref = repo.lookup_reference(refname) if ref.type == GIT_REF_OID: local_refs[ref.peel().id] = refname if not remote_refs: return False, "has no upstream remote branches" if not local_refs: return False, "has no local branches" if not repo.lookup_branch("%s/%s" % (origin.name, project.master_branch), GIT_BRANCH_REMOTE): return False, "has no upstream master branch" for remote_ref in remote_refs: for commit in repo.walk(remote_ref): if commit.id in local_refs: del local_refs[commit.id] if local_refs: return False, "has local commits: %s" % ", ".join( ["'%s'" % name for _, name in iteritems(local_refs)]) return True, ""
def changed_includes(conf=None): from pygit2 import Repository, GIT_STATUS_CURRENT, GIT_STATUS_IGNORED conf = lazy_conf(conf) repo_path = conf.paths.projectroot r = Repository(repo_path) changed = [] for path, flag in r.status().items(): if flag not in [GIT_STATUS_CURRENT, GIT_STATUS_IGNORED]: if path.startswith('source/'): if path.endswith('.txt'): changed.append(path[6:]) source_path = os.path.join(conf.paths.source, conf.paths.output, conf.git.branches.current, 'json') changed_report = [] for report in _generate_report(None): if report['source'][len(source_path):] in changed: changed_report.append(report) return changed_report
def changed(output='print'): try: from pygit2 import Repository, GIT_STATUS_CURRENT, GIT_STATUS_IGNORED except ImportError: puts('[stats]: cannot detect changed files. Please install pygit2') conf = get_conf() repo_path = conf.paths.projectroot r = Repository(repo_path) changed = [] for path, flag in r.status().items(): if flag not in [GIT_STATUS_CURRENT, GIT_STATUS_IGNORED]: if path.startswith(conf.paths.source): if path.endswith('.txt'): changed.append(path[6:]) source_path = os.path.join( conf.paths.source, conf.paths.output, conf.git.branches.current, 'json') changed_report = [] for report in generate_report(None): if report['source'][len(source_path):] in changed: changed_report.append(report) if not len(changed_report) == 0: changed_report.append(multi(data=changed_report, output_file=None)) if output is None: return changed_report elif output == 'print': puts(json.dumps(changed_report, indent=2)) else: json.dump(changed_report, output)
class GitRepo: """A class that manages a git repository. This class enables versiong via git for a repository. You can stage and commit files and checkout different commits of the repository. """ path = '' pathspec = [] repo = None callback = None author_name = 'QuitStore' author_email = '*****@*****.**' gcProcess = None def __init__(self, path, origin=None, gc=False): """Initialize a new repository from an existing directory. Args: path: A string containing the path to the repository. origin: The remote URL where to clone and fetch from and push to """ logger = logging.getLogger('quit.core.GitRepo') logger.debug('GitRepo, init, Create an instance of GitStore') self.path = path self.gc = gc if not exists(path): try: makedirs(path) except OSError as e: raise Exception('Can\'t create path in filesystem:', path, e) try: self.repo = Repository(path) except KeyError: pass except AttributeError: pass if origin: self.callback = QuitRemoteCallbacks() if self.repo: if self.repo.is_bare: raise QuitGitRepoError('Bare repositories not supported, yet') if origin: # set remote self.addRemote('origin', origin) else: if origin: # clone self.repo = self.cloneRepository(origin, path, self.callback) else: self.repo = init_repository(path=path, bare=False) def cloneRepository(self, origin, path, callback): try: repo = clone_repository(url=origin, path=path, bare=False, callbacks=callback) return repo except Exception as e: raise QuitGitRepoError( "Could not clone from: {} origin. {}".format(origin, e)) def addall(self): """Add all (newly created|changed) files to index.""" self.repo.index.read() self.repo.index.add_all(self.pathspec) self.repo.index.write() def addfile(self, filename): """Add a file to the index. Args: filename: A string containing the path to the file. """ index = self.repo.index index.read() try: index.add(filename) index.write() except Exception as e: logger.info( "GitRepo, addfile, Could not add file {}.".format(filename)) logger.debug(e) def addRemote(self, name, url): """Add a remote. Args: name: A string containing the name of the remote. url: A string containing the url to the remote. """ try: self.repo.remotes.create(name, url) logger.info("Successfully added remote: {} - {}".format(name, url)) except Exception as e: logger.info("Could not add remote: {} - {}".format(name, url)) logger.debug(e) try: self.repo.remotes.set_push_url(name, url) self.repo.remotes.set_url(name, url) except Exception as e: logger.info("Could not set push/fetch urls: {} - {}".format( name, url)) logger.debug(e) def checkout(self, commitid): """Checkout a commit by a commit id. Args: commitid: A string cotaining a commitid. """ try: commit = self.repo.revparse_single(commitid) self.repo.set_head(commit.oid) self.repo.reset(commit.oid, GIT_RESET_HARD) logger.info("Checked out commit: {}".format(commitid)) except Exception as e: logger.info("Could not check out commit: {}".format(commitid)) logger.debug(e) def commit(self, message=None): """Commit staged files. Args: message: A string for the commit message. Raises: Exception: If no files in staging area. """ if self.isstagingareaclean(): # nothing to commit return index = self.repo.index index.read() tree = index.write_tree() try: author = Signature(self.author_name, self.author_email) comitter = Signature(self.author_name, self.author_email) if len(self.repo.listall_reference_objects()) == 0: # Initial Commit if message is None: message = 'Initial Commit from QuitStore' self.repo.create_commit('HEAD', author, comitter, message, tree, []) else: if message is None: message = 'New Commit from QuitStore' self.repo.create_commit('HEAD', author, comitter, message, tree, [self.repo.head.get_object().hex]) logger.info('Updates commited') except Exception as e: logger.info('Nothing to commit') logger.debug(e) if self.gc: self.garbagecollection() def commitexists(self, commitid): """Check if a commit id is part of the repository history. Args: commitid: String of a Git commit id. Returns: True, if commitid is part of commit log False, else. """ if commitid in self.getids(): return True else: return False def garbagecollection(self): """Start garbage collection. Args: commitid: A string cotaining a commitid. """ try: # Check if the garbage collection process is still running if self.gcProcess is None or self.gcProcess.poll() is not None: # Start garbage collection with "--auto" option, # which imidietly terminates, if it is not necessary self.gcProcess = Popen(["git", "gc", "--auto", "--quiet"], cwd=self.path) logger.debug('Spawn garbage collection') except Exception as e: logger.debug('Git garbage collection failed to spawn') logger.debug(e) def getpath(self): """Return the path of the git repository. Returns: A string containing the path to the directory of git repo """ return self.path def getcommits(self): """Return meta data about exitsting commits. Returns: A list containing dictionaries with commit meta data """ commits = [] if len(self.repo.listall_reference_objects()) > 0: for commit in self.repo.walk(self.repo.head.target, GIT_SORT_REVERSE): commits.append({ 'id': str(commit.oid), 'message': str(commit.message), 'commit_date': datetime.fromtimestamp( commit.commit_time).strftime('%Y-%m-%dT%H:%M:%SZ'), 'author_name': commit.author.name, 'author_email': commit.author.email, 'parents': [c.hex for c in commit.parents], }) return commits def getids(self): """Return meta data about exitsting commits. Returns: A list containing dictionaries with commit meta data """ ids = [] if len(self.repo.listall_reference_objects()) > 0: for commit in self.repo.walk(self.repo.head.target, GIT_SORT_REVERSE): ids.append(str(commit.oid)) return ids def isgarbagecollectionon(self): """Return if gc is activated or not. Returns: True, if activated False, if not """ return self.gc def isstagingareaclean(self): """Check if staging area is clean. Returns: True, if staginarea is clean False, else. """ status = self.repo.status() for filepath, flags in status.items(): if flags != GIT_STATUS_CURRENT: return False return True def pull(self, remote='origin', branch='master'): """Pull if possible. Return: True: If successful. False: If merge not possible or no updates from remote. """ try: self.repo.remotes[remote].fetch() except Exception as e: logger.info("Can not pull: Remote {} not found.".format(remote)) logger.debug(e) ref = 'refs/remotes/' + remote + '/' + branch remoteid = self.repo.lookup_reference(ref).target analysis, _ = self.repo.merge_analysis(remoteid) if analysis & GIT_MERGE_ANALYSIS_UP_TO_DATE: # Already up-to-date pass elif analysis & GIT_MERGE_ANALYSIS_FASTFORWARD: # fastforward self.repo.checkout_tree(self.repo.get(remoteid)) master_ref = self.repo.lookup_reference('refs/heads/master') master_ref.set_target(remoteid) self.repo.head.set_target(remoteid) elif analysis & GIT_MERGE_ANALYSIS_NORMAL: self.repo.merge(remoteid) tree = self.repo.index.write_tree() msg = 'Merge from ' + remote + ' ' + branch author = Signature(self.author_name, self.author_email) comitter = Signature(self.author_name, self.author_email) self.repo.create_commit('HEAD', author, comitter, msg, tree, [self.repo.head.target, remoteid]) self.repo.state_cleanup() else: logger.debug('Can not pull. Unknown merge analysis result') def push(self, remote='origin', branch='master'): """Push if possible. Return: True: If successful. False: If diverged or nothing to push. """ ref = ['refs/heads/' + branch] try: remo = self.repo.remotes[remote] except Exception as e: logger.info( "Can not push. Remote: {} does not exist.".format(remote)) logger.debug(e) return try: remo.push(ref, callbacks=self.callback) except Exception as e: logger.info("Can not push to {} with ref {}".format( remote, str(ref))) logger.debug(e) def getRemotes(self): remotes = {} try: for remote in self.repo.remotes: remotes[remote.name] = [remote.url, remote.push_url] except Exception as e: logger.info('No remotes found.') logger.debug(e) return {} return remotes
class GitStorage(Storage): """ Git file storage backend. """ def __init__(self, path): """ Initialize repository. :param path: Absolute path to the existing Git repository. :type path: str """ super(GitStorage, self).__init__() self.repo = Repository(path) self.index = self.repo.index self.index.read() @classmethod def create_storage(cls, path): """ Create repository, and return GitStorage object on it :param path: Absolute path to the Git repository to create. :type path: str :returns: GitStorage """ init_repository(path, False) return cls(path) def commit(self, user, message): """ Save previous changes in a new commit. :param user: The commit author/committer. :type user: django.contrib.auth.models.User :param message: The commit message. :type message: unicode :returns: pygit2.Commit """ # Refresh index before committing index = self.repo.index index.read() # Check the status of the repository status = self.repo.status() for filename, flags in status.items(): # the file was deleted if flags in (GIT_STATUS_INDEX_DELETED, GIT_STATUS_WT_DELETED): # remove it from the tree del index[filename] # or the file was modified/added elif flags in (GIT_STATUS_INDEX_MODIFIED, GIT_STATUS_INDEX_NEW, GIT_STATUS_WT_MODIFIED, GIT_STATUS_WT_NEW): # add it to the tree index.add(filename) treeid = index.write_tree() # Now make the commit author = Signature(u'{0} {1}'.format( user.first_name, user.last_name).encode('utf-8'), user.email.encode('utf-8') ) committer = author try: parents = [self.repo.head.oid] except GitError: parents = [] commit = self.repo.create_commit( 'refs/heads/master', author, committer, message, treeid, parents ) # Write changes to disk index.write() # and refresh index. self.index.read() # Return commit object return self.repo[commit] def log(self, name=None, limit=10): """ Get history of the repository, or of a file if name is not None. :param name: File name within the repository. :type name: unicode or None :param limit: Maximal number of commits to get (default: 10), use a negative number to get all. :type limit: int :returns: list of pygit2.Commit """ commits = [] if not name: # Look for `limit` commits for commit in self.repo.walk(self.repo.head.oid, GIT_SORT_TIME): commits.append(commit) limit = limit - 1 if limit == 0: break else: # For each commits for commit in self.repo.walk(self.repo.head.oid, GIT_SORT_TIME): # Check the presence of the file in the tree if commit.parents: # If the commit has parents, check if the file is present # in the diff diff = commit.tree.diff(commit.parents[0].tree) for patch in diff: # If the filename is the patch's filename... if name.encode('utf-8') == patch.new_file_path: # ... then we can add the commit to the list # and leave the loop commits.append(commit) limit = limit - 1 break else: # But if the commit has no parents (root commit) # Simply check in its tree try: commit.tree[name] # no error raised, it means the entry exists, so add the # commit to the list commits.append(commit) limit = limit - 1 # If the file is not in the tree, then it raises a KeyError, # so, just ignore it. except KeyError: pass # If the limit is reached, leave the loop if limit == 0: break return commits def diffs(self, name=None, limit=10): """ Get diffs between commits. Return the following dict : {"diffs": [ { "msg": unicode(<commit message>), "date": datetime.fromtimestamp(<commit date>), "author": unicode(<author name>), "sha": unicode(<commit SHA>), "parent_sha": unicode(<parent commit SHA>), # optional }, # ... ]} :param name: File name within the repository. :type name: unicode or None :param limit: Maximal number of diffs to get (default: 10), use a negative number to get all. :type limit: int :returns: dict """ commits = self.log(name=name, limit=limit) diffs = {'diffs': []} # For each commit for commit in commits: # Create a JSON object containing informations about the commit diff = { 'msg': commit.message, 'date': datetime.datetime.fromtimestamp(commit.commit_time), 'author': commit.author.name, 'sha': commit.hex, } if commit.parents: diff['parent_sha'] = commit.parents[0].hex # The SHA and parent SHA will be used to get the diff via AJAX. diffs['diffs'].append(diff) return diffs def diff(self, asha, bsha, name=None): """ Get diff between two commits. :param asha: SHA of commit A. :type asha: unicode :param bsha: SHA of commit B. :type bsha: unicode :param name: File name within the repository. :type name: unicode or None :returns: unicode """ c1 = self.repo[asha] c2 = self.repo[bsha] d = c1.tree.diff(c2.tree) if name: diff = u'' # For each patch in the diff for patch in d: # Check if the patch is our file if name.encode('utf-8') == patch.new_file_path: # Format the patch for hunk in patch.hunks: p = u'\n'.join(hunk.lines) # And add the diff to the final diff diff = u'{0}{1}'.format(diff, p) return diff # For a global diff, just return the full patch else: return d.patch def search(self, pattern, exclude=None): """ Search pattern in GIT repository. :param pattern: Pattern to search. :type pattern: unicode :param exclude: Exclude some files from the search results :type exclude: regex :returns: list of tuple containing the filename and the list of matched lines. """ entries = [] self.index.read() # For each files in the index for ientry in self.index: # If the filename match the exclude_file regex, then ignore it if exclude and re.match(exclude, ientry.path.decode('utf-8')): continue # Get the associated blob blob = self.repo[ientry.oid] # Create entry entry = (ientry.path.decode('utf-8'), []) # Add matched lines to the entry for line in blob.data.decode('utf-8').splitlines(): if pattern in line: entry[1].append(line) # If the entry has no matched lines, then ignore if entry[1]: entries.append(entry) return entries def is_dir(self, name): """ Check if name refers to a directory. :param name: File name within the repository. :type name: unicode :returns: True, False """ # Check if the path exists, if not returns default value. if not self.exists(name): return False # Get the TreeEntry associated to name tentry = self.repo.head.tree[name] # Convert it to its pygit2 representation obj = tentry.to_object() # If it's a Tree, then we can return True if isinstance(obj, Tree): return True # The instance is a Blob, so it's a file, return False else: return False def mimetype(self, name): """ Get the mimetype of a file. :param name: File name within the repository. :type name: unicode :returns: str """ # If the file is a directory if self.is_dir(name): return 'inode/directory' # Or doesn't exist elif not self.exists(name): return 'unknown' # The file exists, check its mimetype else: import urllib import mimetypes url = urllib.pathname2url(name.encode('utf-8')) return mimetypes.guess_type(url)[0] or 'unknown' def walk(self): """ Walk through the repository. """ self.index.read() for entry in self.index: yield entry # Storage API def accessed_time(self, name): """ Get last accessed time of a file. :param name: File name within the repository. :type name: unicode :returns: datetime :raises: IOError """ if not self.exists(name): raise IOError(u"{0}: Not found in repository".format(name)) abspath = os.path.join(self.repo.workdir, name) stats = os.stat(abspath) return datetime.datetime.fromtimestamp(stats.st_atime) def created_time(self, name): """ Get creation time of a file. :param name: File name within the repository. :type name: unicode :returns: datetime :raises: IOError """ if not self.exists(name): raise IOError(u"{0}: Not found in repository".format(name)) abspath = os.path.join(self.repo.workdir, name) stats = os.stat(abspath) return datetime.datetime.fromtimestamp(stats.st_ctime) def modified_time(self, name): """ Get last modified time of a file. :param name: File name within the repository. :type name: unicode :returns: datetime :raises: IOError """ if not self.exists(name): raise IOError(u"{0}: Not found in repository".format(name)) abspath = os.path.join(self.repo.workdir, name) stats = os.stat(abspath) return datetime.datetime.fromtimestamp(stats.st_mtime) def size(self, name): """ Get file's size. :param name: File name within the repository. :type name: unicode :returns: int :raises: IOError """ if not self.exists(name): raise IOError(u"{0}: Not found in repository".format(name)) e = self.index[name] blob = self.repo[e.oid] return blob.size def exists(self, path): """ Check if ``path`` exists in the Git repository. :param path: Path within the repository of the file to check. :type param: unicode :returns: True if the file exists, False if the name is available for a new file. """ # If the head is orphaned (does not point to any commit), returns False # because there is nothing in the repository. if self.repo.head_is_orphaned: return False # Try getting the path via the tree try: entry = self.repo.head.tree[path] return True # If it raises a KeyError, then the path doesn't exist except KeyError: return False def listdir(self, path=None): """ Lists the contents of the specified path. :param path: Path of the directory to list (or None to list the root). :type path: unicode or None :returns: a 2-tuple of lists; the first item being directories, the second item being files. """ abspath = os.path.join(self.repo.workdir, path) if path else self.repo.workdir dirs = [] files = [] for e in os.listdir(abspath): entry_fullpath = os.path.join(abspath, e) if os.path.isdir(entry_fullpath): if e != '.git': dirs.append(e.decode('utf-8')) else: files.append(e.decode('utf-8')) return (dirs, files) def open(self, name, mode='rb'): """ Opens the file given by name. :param name: Name of the file to open. :type name: unicode :param mode: Flags for openning the file (see builtin ``open`` function). :type mode: str :returns: GitFile """ abspath = os.path.join(self.repo.workdir, name) dirname = os.path.dirname(abspath) if 'w' in mode and not os.path.exists(dirname): os.makedirs(dirname) return GitFile(open(abspath, mode)) def path(self, name): """ Return the absolute path of the file ``name`` within the repository. :param name: Name of the file within the repository. :type name: unicode :returns: str :raises: IOError """ if not self.exists(name): raise IOError(u"{0}: Not found in repository".format(name)) e = self.index[name] return os.path.join(self.repo.workdir, e.path).decode('utf-8') def save(self, name, content): """ Saves a new file using the storage system, preferably with the name specified. If there already exists a file with this name, the storage system may modify the filename as necessary to get a unique name. The actual name of the stored file will be returned. :param name: Name of the new file within the repository. :type name: unicode :param content: Content to save. :type content: django.core.files.File :returns: str """ new_name = self.get_available_name(name) abspath = os.path.join(self.repo.workdir, new_name) dirname = os.path.dirname(abspath) if not os.path.exists(dirname): os.makedirs(dirname) with open(abspath, 'wb') as f: for chunk in content.chunks(): f.write(chunk) def delete(self, name): """ Deletes the file referenced by name. :param name: Name of the file within the repository to delete :type name: unicode :raises: IOError """ if not self.exists(name): raise IOError(u"{0}: Not found in repository".format(name)) abspath = os.path.join(self.repo.workdir, name) os.remove(abspath)
class RepositoryInfo(object): """ wraps an pygit2.Repository object """ def __init__(self, repo_path): self._repo = Repository(repo_path) self.count_unmodified = 0 self.count_wt_modified = 0 self.count_wt_new = 0 self.count_wt_deleted = 0 self.count_index_modified = 0 self.count_index_new = 0 self.count_index_deleted = 0 self._count() @property def path(self): sep = '/' splitted = self._repo.path.split(sep)[0:-2] return sep.join(splitted) @property def has_workingtree_changes(self): return self.count_wt_deleted > 0 or self.count_wt_modified > 0 or self.count_wt_new > 0 @property def has_index_changes(self): return self.count_index_deleted > 0 or self.count_index_modified > 0 or self.count_index_new > 0 def _count(self): _status = self._repo.status() for file_path, flags in _status.items(): if flags == GIT_STATUS_CURRENT: self.count_unmodified += 1 elif flags == GIT_STATUS_WT_MODIFIED: self.count_wt_modified += 1 elif flags == GIT_STATUS_WT_NEW: self.count_wt_new += 1 elif flags == GIT_STATUS_INDEX_NEW: self.count_index_new += 1 elif flags == GIT_STATUS_INDEX_MODIFIED: self.count_index_modified += 1 elif flags == GIT_STATUS_INDEX_DELETED: self.count_index_deleted += 1 elif flags == GIT_STATUS_WT_DELETED: self.count_wt_deleted += 1 @property def current_branch_name(self): # ToDo: Why does self._repo.head.shorthand not work? head = self._repo.head head_name = head.name.split('/')[-1:] return head_name[0] @property def is_head_upstream_branch(self): """ determines if current head is the same commit as the remote commit """ if self._repo.head_is_detached: return False current_branch_name = self.current_branch_name head = self._repo.head remote_branch = self._repo.lookup_branch(current_branch_name).upstream if remote_branch: return remote_branch.target.hex == head.target.hex return False @property def is_head_detached(self): return self._repo.head_is_detached
class GitInfo: ''' Call .current_oid_hex to save commit-oid as str. ''' def __init__(self, base_path=None): self.base_path = None self.repo = None self.dirty = None self.ignored = None self.status = None self.diff = None self.head_name = None self.current_commit = None self.current_oid_hex = None if base_path is not None: self.open(base_path) def open(self, base_path): self.base_path = base_path repo_path = discover_repository(self.base_path) self.repo = Repository(repo_path) try: self.repo.head.target except Exception as e: raise Exception('repo.head not found! No commit in repo.') def collect(self): '''Compare worktree to `HEAD`. Show difference, and HEAD commit info. No branches check. ''' if self.repo is None: repo_path = discover_repository(self.base_path) self.repo = Repository(repo_path) # diff # [patch,patch,...] # patch.text :str self.diff = list(self.repo.diff('HEAD')) # status self.status = self.repo.status() ss = self.status stt_p = [] stt_p_ig = [] stt_f = [] for filepath, flag in ss.items(): flag = statusFlag(flag) if 'GIT_STATUS_IGNORED' in flag: stt_p_ig.append(filepath) else: stt_p.append(filepath) stt_f.append(flag) self.dirty = [(p, f) for p, f in zip(stt_p, stt_f)] self.ignored = stt_p_ig # branch,ref self.head_name = self.repo.head.name # commit # commit attrs: # author,commiter: Signature: .name .email # commit_time # Unixtimestamp self.current_commit = self.repo.head.peel() self.current_oid_hex = self.current_commit.oid.hex def report(self): self.collect() data = {} #diff = [patch.text for patch in self.diff] #data['diff']= diff status = {'dirty': self.dirty, 'ignored': self.ignored} data['status'] = status ref = self.head_name data['head'] = ref c = self.current_commit t = c.commit_time utc_str = datetime.utcfromtimestamp(t).strftime('%Y-%m-%d %H:%M:%S') commit = { 'message': c.message, 'oid': c.oid, 'time': utc_str, 'author': (c.author.name, c.author.email), 'committer': (c.committer.name, c.author.email), } data['commit'] = commit return data def commit(self, msg): return commitAll(self.repo, msg, is_first=False) def __str__(self): return str(self.report())
def show_status(srcdir, packages, projects, other_git, ws_state, show_up_to_date=True, cache=None): def create_upstream_status(repo, head_branch, master_branch, master_remote_branch, tracking_branch): status = [] if not repo.head_is_detached and not has_pending_merge(repo): if tracking_branch is not None: if master_remote_branch is not None: if tracking_branch.remote_name != master_remote_branch.remote_name: status.append("@!@{rf}remote '%s'" % tracking_branch.remote_name) if need_push(repo, head_branch): status.append("@!@{yf}needs push") elif need_pull(repo, head_branch): status.append("@!@{cf}needs pull") elif not is_up_to_date(repo, head_branch): status.append("@!@{yf}needs pull -M") else: if head_branch: status.append("@!on branch '%s'" % repo.head.shorthand) else: status.append("empty branch") if master_remote_branch is None: status.append("@!@{rf}no remote") elif master_branch is None: status.append("@!@{rf}untracked remote") if is_up_to_date(repo, master_branch) or need_push(repo, master_branch): if need_pull(repo, head_branch, master_branch): status.append("@!@{cf}needs pull -L") else: if not is_ancestor(repo, master_branch, head_branch): status.append("@!@{yf}needs merge --from-master") if not is_up_to_date(repo, head_branch, master_branch): status.append("@!@{yf}needs merge --to-master") if master_branch is not None and master_remote_branch is not None and (tracking_branch is None or tracking_branch.name != master_remote_branch.name): if need_push(repo, master_branch): status.append("@!@{yf}%s needs push" % master_branch.shorthand) elif need_pull(repo, master_branch): status.append("@!@{cf}%s needs pull" % master_branch.shorthand) elif not is_up_to_date(repo, master_branch): status.append("@!@{yf}%s needs merge" % master_branch.shorthand) return status def create_local_status(repo, upstream_status, is_dirty): status = [] if repo.head_is_detached: status.append("@!@{rf}detached HEAD") return status if has_pending_merge(repo): if repo.index.conflicts: status.append("@!@{rf}merge conflicts") else: status.append("@!@{yf}merged, needs commit") return status if is_dirty: status.append("@!@{yf}needs commit") status += upstream_status if not status: if not show_up_to_date: return None status.append("@!@{gf}up-to-date") return status table = TableView("Package", "Path", "Status") found_packages = set() for project in projects: repo = Repository(os.path.join(srcdir, project.workspace_path, ".git")) dirty_files = [a for a, b in iteritems(repo.status()) if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT] head_branch = get_head_branch(repo) tracking_branch = head_branch.upstream if head_branch else None master_remote = get_origin(repo, project) if master_remote is not None: master_remote_branch = repo.lookup_branch("%s/%s" % (master_remote.name, project.master_branch), GIT_BRANCH_REMOTE) master_branch = None if master_remote_branch is not None: for name in repo.listall_branches(GIT_BRANCH_LOCAL): b = repo.lookup_branch(name, GIT_BRANCH_LOCAL) if b.upstream and b.upstream.branch_name == master_remote_branch.branch_name: master_branch = b break else: master_remote_branch = None master_branch = None ws_packages = find_catkin_packages(srcdir, project.workspace_path, cache=cache) found_packages |= set(ws_packages.keys()) upstream_status = create_upstream_status(repo, head_branch, master_branch, master_remote_branch, tracking_branch) for name, pkg_list in iteritems(ws_packages): if name not in packages: continue for pkg in pkg_list: is_dirty = False local_path = os.path.relpath(pkg.workspace_path, project.workspace_path) if dirty_files and local_path == ".": is_dirty = True else: for fpath in dirty_files: if path_has_prefix(fpath, local_path): is_dirty = True break status = create_local_status(repo, upstream_status, is_dirty) if status is not None: head, tail = os.path.split(pkg.workspace_path) pkg_path = escape(head + "/" if tail == name else pkg.workspace_path) table.add_row(escape(name), pkg_path, status) for path in other_git: repo = Repository(os.path.join(srcdir, path, ".git")) dirty_files = [a for a, b in iteritems(repo.status()) if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT] head_branch = get_head_branch(repo) tracking_branch = head_branch.upstream if head_branch else None ws_packages = find_catkin_packages(srcdir, path, cache=cache) found_packages |= set(ws_packages.keys()) upstream_status = create_upstream_status(repo, head_branch, None, None, tracking_branch) for name, pkg_list in iteritems(ws_packages): if name not in packages: continue for pkg in pkg_list: is_dirty = False local_path = os.path.relpath(pkg.workspace_path, path) if dirty_files and local_path == ".": is_dirty = True else: for fpath in dirty_files: if path_has_prefix(fpath, local_path): is_dirty = True break status = create_local_status(repo, upstream_status, is_dirty) if status is not None: head, tail = os.path.split(pkg.workspace_path) pkg_path = escape(head + "/" if tail == name else pkg.workspace_path) table.add_row(escape(name), pkg_path, status) missing = set(packages) - found_packages for name in missing: path_list = [] status = "no git" if name in ws_state.ws_packages: for pkg in ws_state.ws_packages[name]: if not os.path.isdir(os.path.join(srcdir, pkg.workspace_path)): status = "@{rf}deleted" head, tail = os.path.split(pkg.workspace_path) path_list.append(escape(head + "/" if tail == name else pkg.workspace_path)) table.add_row(escape(name), path_list, status) if table.empty(): if found_packages: msg("Everything is @!@{gf}up-to-date@|.\n") else: warning("no Git repositories\n") else: table.sort(0) table.write(sys.stdout)
class GitBlack: def __init__(self): self.repo = Repository(".") self.patchers = {} def get_blamed_deltas(self, patch): filename = patch.delta.old_file.path self.patchers[filename] = Patcher(self.repo, filename) hb = HunkBlamer(self.repo, patch) return hb.blames() def group_blame_deltas(self, blames): for delta_blame in blames: commits = tuple(sorted(delta_blame.commits)) self.grouped_deltas.setdefault(commits, []).append(delta_blame.delta) self.progress += 1 now = time.monotonic() if now - self.last_log > 0.04: sys.stdout.write("Reading file {}/{} \r".format( self.progress, self.total)) sys.stdout.flush() self.last_log = now def commit_changes(self): start = time.monotonic() self.grouped_deltas = {} for path, status in self.repo.status().items(): if status & index_statuses: raise GitIndexNotEmpty patches = [] self._file_modes = {} diff = self.repo.diff(context_lines=0, flags=GIT_DIFF_IGNORE_SUBMODULES) for patch in diff: if patch.delta.status != GIT_DELTA_MODIFIED: continue self._file_modes[ patch.delta.old_file.path] = patch.delta.old_file.mode patches.append(patch) self.progress = 0 self.last_log = 0 self.total = len(patches) executor = ThreadPoolExecutor(max_workers=8) tasks = set() for patch in patches: tasks.add(executor.submit(self.get_blamed_deltas, patch)) if len(tasks) > 8: done, not_done = wait(tasks, return_when=FIRST_COMPLETED) for task in done: self.group_blame_deltas(task.result()) tasks -= set(done) for task in tasks: self.group_blame_deltas(task.result()) secs = time.monotonic() - start sys.stdout.write("Reading file {}/{} ({:.2f} secs).\n".format( self.progress, self.total, secs)) start = time.monotonic() self.total = len(self.grouped_deltas) self.progress = 0 self.last_log = 0 for commits, deltas in self.grouped_deltas.items(): blobs = self._create_blobs(deltas) self._commit(commits, blobs) secs = time.monotonic() - start print("Making commit {}/{} ({:.2f} secs).".format( self.progress, self.total, secs)) def _create_blobs(self, deltas): filenames = set() for delta in deltas: self.patchers[delta.filename].apply(delta) filenames.add(delta.filename) blobs = {} for filename in filenames: blob_id = self.repo.create_blob(self.patchers[filename].content()) blobs[filename] = blob_id return blobs def _commit(self, original_commits, blobs): for filename, blob_id in blobs.items(): file_mode = self._file_modes[filename] index_entry = IndexEntry(filename, blob_id, file_mode) self.repo.index.add(index_entry) commits = [self.repo.get(h) for h in original_commits] main_commit = commits[0] if len(commits) > 1: # most recent commit main_commit = sorted(commits, key=commit_datetime)[-1] commit_message = main_commit.message commit_message += "\n\nautomatic commit by git-black, original commits:\n" commit_message += "\n".join( [" {}".format(c) for c in original_commits]) committer = Signature( name=self.repo.config["user.name"], email=self.repo.config["user.email"], ) self.repo.index.write() tree = self.repo.index.write_tree() head = self.repo.head.peel() self.repo.create_commit("HEAD", main_commit.author, committer, commit_message, tree, [head.id]) self.progress += 1 now = time.monotonic() if now - self.last_log > 0.04: sys.stdout.write("Making commit {}/{} \r".format( self.progress, self.total)) sys.stdout.flush() self.last_log = now
def show_status(srcdir, packages, projects, other_git, ws_state, show_up_to_date=True, cache=None): def create_upstream_status(repo, head_branch, master_branch, master_remote_branch, tracking_branch): status = [] if not repo.head_is_detached and not has_pending_merge(repo): if tracking_branch is not None: if master_remote_branch is not None: if tracking_branch.remote_name != master_remote_branch.remote_name: status.append("@!@{rf}remote '%s'" % tracking_branch.remote_name) if need_push(repo, head_branch): status.append("@!@{yf}needs push") elif need_pull(repo, head_branch): status.append("@!@{cf}needs pull") elif not is_up_to_date(repo, head_branch): status.append("@!@{yf}needs pull -M") else: if head_branch: status.append("@!on branch '%s'" % repo.head.shorthand) else: status.append("empty branch") if master_remote_branch is None: status.append("@!@{rf}no remote") elif master_branch is None: status.append("@!@{rf}untracked remote") if is_up_to_date(repo, master_branch) or need_push( repo, master_branch): if need_pull(repo, head_branch, master_branch): status.append("@!@{cf}needs pull -L") else: if not is_ancestor(repo, master_branch, head_branch): status.append("@!@{yf}needs merge --from-master") if not is_up_to_date(repo, head_branch, master_branch): status.append("@!@{yf}needs merge --to-master") if master_branch is not None and master_remote_branch is not None and ( tracking_branch is None or tracking_branch.name != master_remote_branch.name): if need_push(repo, master_branch): status.append("@!@{yf}%s needs push" % master_branch.shorthand) elif need_pull(repo, master_branch): status.append("@!@{cf}%s needs pull" % master_branch.shorthand) elif not is_up_to_date(repo, master_branch): status.append("@!@{yf}%s needs merge" % master_branch.shorthand) return status def create_local_status(repo, upstream_status, is_dirty): status = [] if repo.head_is_detached: status.append("@!@{rf}detached HEAD") return status if has_pending_merge(repo): if repo.index.conflicts: status.append("@!@{rf}merge conflicts") else: status.append("@!@{yf}merged, needs commit") return status if is_dirty: status.append("@!@{yf}needs commit") status += upstream_status if not status: if not show_up_to_date: return None status.append("@!@{gf}up-to-date") return status table = TableView("Package", "Path", "Status") found_packages = set() for project in projects: repo = Repository(os.path.join(srcdir, project.workspace_path, ".git")) dirty_files = [ a for a, b in iteritems(repo.status()) if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT ] head_branch = get_head_branch(repo) tracking_branch = head_branch.upstream if head_branch else None master_remote = get_origin(repo, project) if master_remote is not None: master_remote_branch = repo.lookup_branch( "%s/%s" % (master_remote.name, project.master_branch), GIT_BRANCH_REMOTE) master_branch = None if master_remote_branch is not None: for name in repo.listall_branches(GIT_BRANCH_LOCAL): b = repo.lookup_branch(name, GIT_BRANCH_LOCAL) if b.upstream and b.upstream.branch_name == master_remote_branch.branch_name: master_branch = b break else: master_remote_branch = None master_branch = None ws_packages = find_catkin_packages(srcdir, project.workspace_path, cache=cache) found_packages |= set(ws_packages.keys()) upstream_status = create_upstream_status(repo, head_branch, master_branch, master_remote_branch, tracking_branch) for name, pkg_list in iteritems(ws_packages): if name not in packages: continue for pkg in pkg_list: is_dirty = False local_path = os.path.relpath(pkg.workspace_path, project.workspace_path) if dirty_files and local_path == ".": is_dirty = True else: for fpath in dirty_files: if path_has_prefix(fpath, local_path): is_dirty = True break status = create_local_status(repo, upstream_status, is_dirty) if status is not None: head, tail = os.path.split(pkg.workspace_path) pkg_path = escape(head + "/" if tail == name else pkg.workspace_path) table.add_row(escape(name), pkg_path, status) for path in other_git: repo = Repository(os.path.join(srcdir, path, ".git")) dirty_files = [ a for a, b in iteritems(repo.status()) if b != GIT_STATUS_IGNORED and b != GIT_STATUS_CURRENT ] head_branch = get_head_branch(repo) tracking_branch = head_branch.upstream if head_branch else None ws_packages = find_catkin_packages(srcdir, path, cache=cache) found_packages |= set(ws_packages.keys()) upstream_status = create_upstream_status(repo, head_branch, None, None, tracking_branch) for name, pkg_list in iteritems(ws_packages): if name not in packages: continue for pkg in pkg_list: is_dirty = False local_path = os.path.relpath(pkg.workspace_path, path) if dirty_files and local_path == ".": is_dirty = True else: for fpath in dirty_files: if path_has_prefix(fpath, local_path): is_dirty = True break status = create_local_status(repo, upstream_status, is_dirty) if status is not None: head, tail = os.path.split(pkg.workspace_path) pkg_path = escape(head + "/" if tail == name else pkg.workspace_path) table.add_row(escape(name), pkg_path, status) missing = set(packages) - found_packages for name in missing: path_list = [] status = "no git" if name in ws_state.ws_packages: for pkg in ws_state.ws_packages[name]: if not os.path.isdir(os.path.join(srcdir, pkg.workspace_path)): status = "@{rf}deleted" head, tail = os.path.split(pkg.workspace_path) path_list.append( escape(head + "/" if tail == name else pkg.workspace_path)) table.add_row(escape(name), path_list, status) if table.empty(): if found_packages: msg("Everything is @!@{gf}up-to-date@|.\n") else: warning("no Git repositories\n") else: table.sort(0) table.write(sys.stdout)
def test_011_commit(root_repo: pygit2.Repository) -> None: core.init() core.commit() assert root_repo.status() == {}