def from_note(klass, repo, note): if type(note) is _Note: note = note.content blob = git.Blob(repo.repo, binascii.a2b_hex(note['blob']), path=note['path']) return klass(repo, blob)
def _create_blob_from_file(repo, filepath): from git.index.fun import stat_mode_to_index_mode from git.util import to_native_path_linux absfilepath = os.path.join(repo.working_tree_dir, filepath) st = os.lstat(absfilepath) blob = git.Blob(repo=repo, binsha=git.Blob.NULL_BIN_SHA, mode=stat_mode_to_index_mode(st.st_mode), path=to_native_path_linux(filepath)) blob = _git_raw_write_object(repo, blob) return blob
def extract_options(repo, blob_id): from git.util import hex_to_bin blob = git.Blob(repo, hex_to_bin(blob_id)) content = blob.data_stream.stream options = set() for line in iter(content.readline, b''): line = line.decode('utf-8').strip() if re_option.match(line): line = content.readline().decode('utf-8').strip() options.add(line.split(';')[0]) return options
def add_file(self, *path, **kwargs): deferred = kwargs.pop('deferred', False) if deferred: stream = StringIO() yield stream else: with super(GitIndexStatsFileSystem, self).add_file(*path) as f: yield f if deferred: data = stream.getvalue() istream = self._repo.odb.store( gitdb.IStream('blob', len(data), six.BytesIO(data.encode()))) blob = git.Blob(self._repo, istream.binsha, git.Blob.file_mode, self._rel_file_path(*path)) self._deferred_entries.append(git.IndexEntry.from_blob(blob)) else: self._added_files.append(self._normalize_path(os.path.join(*path))) self._dirty = True
def object_index(self): if self._object_index is None: logger.debug("building index from git notes") self._object_index = {} for line in self.git.notes('list').splitlines(): # accessing note blob directly is much faster than running 'git notes show' note_id, note_target = line.split() note_blob = git.Blob(self.repo, binascii.a2b_hex(note_id)) note_content = note_blob.data_stream.read() note_obj = json.loads(note_content) note_class = note_obj["class"] note_id = note_obj["id"] if note_class not in self._object_index: self._object_index[note_class] = {} if note_id not in self._object_index[note_class]: self._object_index[note_class][note_id] = note_obj else: logger.warn( f"Found duplicates for {note_class} id {note_id}: {self._object_index[note_class][note_id]['path']}" ) return self._object_index
def add_file(repo, fname, content): istream = gitdb.IStream("blob", len(content), BytesIO(content)) repo.odb.store(istream) blob = git.Blob(repo, istream.binsha, 0o100644, fname) repo.index.add([git.IndexEntry.from_blob(blob)])
def from_note(cls, repo, note): if isinstance(note, _Note): note = note.content blob = git.Blob(repo.repo, binascii.a2b_hex(note['blob']), path=note['path']) return cls(repo, blob)
def merge_libraries (self, * names): num_merged = 0 num_failed = 0 for library_name, library_data \ in self.third_party_index.items (): if names and library_name not in names: continue if not "merge" in library_data: continue library_path = ( "%s/third-party/%s" % ( self.project_path, library_name)) try: for library_merge in library_data ["merge"]: with log.status ( "Merging library: %s (%s -> %s)" % ( library_name, library_merge ["source"], library_merge ["target"])): unmerged_path = ( ".merged/%s" % ( library_merge ["target"])) if os.path.exists ( "%s/%s" % ( self.project_path, unmerged_path)): unmerged_tree = ( self.git_repo.head.commit.tree [ unmerged_path]) else: unmerged_tree = ( self.git_repo.tree ( "4b825dc642cb6eb9a060e54bf8d69288fbee4904")) target_path = ( "%s" % ( library_merge ["target"] [1:])) if os.path.exists ( "%s/%s" % ( self.project_path, target_path)): target_tree = ( self.git_repo.head.commit.tree [ target_path]) else: target_tree = ( self.git_repo.tree ( "4b825dc642cb6eb9a060e54bf8d69288fbee4904")) source_path = ( "third-party/%s%s" % ( library_name, library_merge ["source"])) source_tree = ( self.git_repo.head.commit.tree [ source_path]) log.notice ( "SOURCE TREE: " + unicode (source_tree)) def expand_parents (item): parts = item.rsplit ("/", 1) if len (parts) == 2: return ([] + expand_parents (parts [0]) + ["/" + parts [1]] ) else: return parts includes = set (map ( lambda include: "third-party/%s%s%s" % ( library_name, library_merge ["source"], include), [ item2 for item1 in library_merge.get ("include", []) for item2 in expand_parents (item1) ])) excludes = set (map ( lambda exclude: "third-party/%s%s%s" % ( library_name, library_merge ["source"], exclude), library_merge.get ("exclude", []))) if includes or excludes: source_index = ( git.IndexFile.from_tree ( self.git_repo, "4b825dc642cb6eb9a060e54bf8d69288fbee4904")) def predicate (item, depth): return not excludes \ or item not in excludes def prune (item, depth): return ( includes and item.path not in includes ) or ( excludes and item.path in excludes ) source_prune_length = len ( "third-party/%s%s/" % ( library_name, library_merge ["source"])) for item in source_tree.traverse ( predicate = lambda i, d: True, prune = prune): source_index.add ( [ git.Blob ( self.git_repo, item.binsha, item.mode, item.path [source_prune_length : ]), ]) source_tree = ( source_index.write_tree ()) merged_index = ( git.IndexFile.from_tree ( self.git_repo, unmerged_tree, target_tree, source_tree)) merged_tree = ( merged_index.write_tree ()) log.notice ( "MERGED TREE: " + unicode (merged_tree)) if os.path.exists ( "%s/%s" % ( self.project_path, target_path)): self.git_repo.git.rm ( "--force", "-r", target_path) self.git_repo.git.read_tree ( unicode (merged_tree), "--prefix", target_path) self.git_repo.git.read_tree ( unicode (source_tree), "--prefix", unmerged_path) log.notice ( "INDEX TREE: " + unicode ( self.git_repo.index.write_tree ())) if len (self.git_repo.index.diff ()): self.git_repo.index.commit ( "Auto-merge changes from %s" % ( library_name)) num_merged += 1 except subprocess.CalledProcessError as error: log.notice ( "Merge failed!") log.output ( error.output) num_failed += 1 if num_failed: log.notice ( "Merged %s libraries with %s failures" % ( num_merged, num_failed)) elif num_merged: log.notice ( "Merged %s libraries" % ( num_merged))
def run(self, src_path, dst_path): src = git.Repo(src_path) dst = git.Repo.init(dst_path) children = defaultdict(set) # binsha => set(binsha) threads = list() # [commit] depend = dict() # binsha => set(binsha) for head in src.heads: st = [head.commit] while st: commit = st.pop() if commit.binsha not in depend: depend[commit.binsha] = {parent.binsha for parent in commit.parents} if not commit.parents: threads.append(commit) for c in commit.parents: if c.binsha not in children: st.append(c) children[c.binsha].add(commit) blob_map_cache = dict() commit_binsha_map = dict() # old binsha => new binsha height = dict() # new binsha => height while threads: commit = threads.pop() index = dst.index blobs = set() for item in commit.tree.traverse(): key = item.binsha, item.mode, item.path if item.type == 'blob': if key in blob_map_cache: if blob_map_cache[key] is not None: value = blob_map_cache[key] blobs.add(value) else: res = self.blob_map(item.data_stream, item.mode, item.path) if res is not None: data, mode, path = res istream = dst.odb.store(IStream('blob', len(data), io.BytesIO(data))) value = blob_map_cache[key] = istream.binsha, mode, path blobs.add(value) else: blob_map_cache[key] = None for data, mode, path in self.commit_add(commit): istream = dst.odb.store(IStream('blob', len(data), io.BytesIO(data))) blobs.add((istream.binsha, mode, path)) # remove/add only the differene old_blobs = {(blob[1].binsha, blob[1].mode, blob[1].path) for blob in index.iter_blobs()} to_remove = list(old_blobs - blobs) to_add = list(blobs - old_blobs) for i in range(0, len(to_remove), 128): index.remove([git.Blob(dst, *t) for t in to_remove[i:i + 128]]) for i in range(0, len(to_add), 128): index.add([git.Blob(dst, *t) for t in to_add[i:i + 128]]) parent_commits=[commit_binsha_map[parent.binsha] for parent in commit.parents] message, author, authored_date, author_tz_offset, committer, committed_date, committer_tz_offset = self.commit_map(commit, commit.message, commit.author, commit.authored_date, commit.author_tz_offset, commit.committer, commit.committed_date, commit.committer_tz_offset) author_date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(authored_date)) + ' ' + git.objects.util.altz_to_utctz_str(author_tz_offset) commit_date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(committed_date)) + ' ' + git.objects.util.altz_to_utctz_str(committer_tz_offset) skip_flag = False if self.remove_empty_commits: # detect grandparents min_height = min(height[parent.binsha] for parent in parent_commits) if parent_commits else 0 st = parent_commits[:] grandparents = set() while st: current = st.pop() for grandparent in current.parents: if grandparent.binsha not in grandparents: grandparents.add(grandparent.binsha) if height[grandparent.binsha] > min_height: st.append(grandparent) parent_commits = [parent for parent in parent_commits if parent.binsha not in grandparents] # detect same parents for i in range(len(parent_commits) - 1, -1, -1): if parent_commits[i].binsha in set(parent.binsha for parent in parent_commits[:i]): parent_commits.pop(i) # skip empty commits for parent in parent_commits: if not index.diff(parent): dst_commit = parent skip_flag = True break if not skip_flag: dst_commit = index.commit(message, parent_commits=parent_commits, author=author, committer=committer, author_date=author_date, commit_date=commit_date) commit_binsha_map[commit.binsha] = dst_commit height[dst_commit.binsha] = max(height[parent.binsha] for parent in dst_commit.parents) + 1 if dst_commit.parents else 0 self.progress(commit, dst_commit) for child in children[commit.binsha]: depend[child.binsha].remove(commit.binsha) if not depend[child.binsha]: threads.append(child) for head in src.heads: if not head.name in dst.heads: dst.create_head(head.name) dst.heads[head.name].commit = commit_binsha_map[head.commit.binsha]
def from_note(klass, repo, note): blob = git.Blob(repo.repo, binascii.a2b_hex(note['blob']), path=note['path']) return klass(repo, blob)