def from_note(klass, repo, note):
if type(note) is _Note:
note = note.content
blob = git.Blob(repo.repo,
binascii.a2b_hex(note['blob']),
path=note['path'])
return klass(repo, blob)
def _create_blob_from_file(repo, filepath):
from git.index.fun import stat_mode_to_index_mode
from git.util import to_native_path_linux
absfilepath = os.path.join(repo.working_tree_dir, filepath)
st = os.lstat(absfilepath)
blob = git.Blob(repo=repo,
binsha=git.Blob.NULL_BIN_SHA,
mode=stat_mode_to_index_mode(st.st_mode),
path=to_native_path_linux(filepath))
blob = _git_raw_write_object(repo, blob)
return blob
def extract_options(repo, blob_id):
from git.util import hex_to_bin
blob = git.Blob(repo, hex_to_bin(blob_id))
content = blob.data_stream.stream
options = set()
for line in iter(content.readline, b''):
line = line.decode('utf-8').strip()
if re_option.match(line):
line = content.readline().decode('utf-8').strip()
options.add(line.split(';')[0])
return options
def add_file(self, *path, **kwargs):
deferred = kwargs.pop('deferred', False)
if deferred:
stream = StringIO()
yield stream
else:
with super(GitIndexStatsFileSystem, self).add_file(*path) as f:
yield f
if deferred:
data = stream.getvalue()
istream = self._repo.odb.store(
gitdb.IStream('blob', len(data), six.BytesIO(data.encode())))
blob = git.Blob(self._repo, istream.binsha, git.Blob.file_mode,
self._rel_file_path(*path))
self._deferred_entries.append(git.IndexEntry.from_blob(blob))
else:
self._added_files.append(self._normalize_path(os.path.join(*path)))
self._dirty = True
def object_index(self):
if self._object_index is None:
logger.debug("building index from git notes")
self._object_index = {}
for line in self.git.notes('list').splitlines():
# accessing note blob directly is much faster than running 'git notes show'
note_id, note_target = line.split()
note_blob = git.Blob(self.repo, binascii.a2b_hex(note_id))
note_content = note_blob.data_stream.read()
note_obj = json.loads(note_content)
note_class = note_obj["class"]
note_id = note_obj["id"]
if note_class not in self._object_index:
self._object_index[note_class] = {}
if note_id not in self._object_index[note_class]:
self._object_index[note_class][note_id] = note_obj
else:
logger.warn(
f"Found duplicates for {note_class} id {note_id}: {self._object_index[note_class][note_id]['path']}"
)
return self._object_index
def add_file(repo, fname, content):
istream = gitdb.IStream("blob", len(content), BytesIO(content))
repo.odb.store(istream)
blob = git.Blob(repo, istream.binsha, 0o100644, fname)
repo.index.add([git.IndexEntry.from_blob(blob)])
def from_note(cls, repo, note):
if isinstance(note, _Note):
note = note.content
blob = git.Blob(repo.repo, binascii.a2b_hex(note['blob']),
path=note['path'])
return cls(repo, blob)
def merge_libraries (self, * names):
num_merged = 0
num_failed = 0
for library_name, library_data \
in self.third_party_index.items ():
if names and library_name not in names:
continue
if not "merge" in library_data:
continue
library_path = (
"%s/third-party/%s" % (
self.project_path,
library_name))
try:
for library_merge in library_data ["merge"]:
with log.status (
"Merging library: %s (%s -> %s)" % (
library_name,
library_merge ["source"],
library_merge ["target"])):
unmerged_path = (
".merged/%s" % (
library_merge ["target"]))
if os.path.exists (
"%s/%s" % (
self.project_path,
unmerged_path)):
unmerged_tree = (
self.git_repo.head.commit.tree [
unmerged_path])
else:
unmerged_tree = (
self.git_repo.tree (
"4b825dc642cb6eb9a060e54bf8d69288fbee4904"))
target_path = (
"%s" % (
library_merge ["target"] [1:]))
if os.path.exists (
"%s/%s" % (
self.project_path,
target_path)):
target_tree = (
self.git_repo.head.commit.tree [
target_path])
else:
target_tree = (
self.git_repo.tree (
"4b825dc642cb6eb9a060e54bf8d69288fbee4904"))
source_path = (
"third-party/%s%s" % (
library_name,
library_merge ["source"]))
source_tree = (
self.git_repo.head.commit.tree [
source_path])
log.notice (
"SOURCE TREE: " + unicode (source_tree))
def expand_parents (item):
parts = item.rsplit ("/", 1)
if len (parts) == 2:
return ([]
+ expand_parents (parts [0])
+ ["/" + parts [1]]
)
else:
return parts
includes = set (map (
lambda include: "third-party/%s%s%s" % (
library_name,
library_merge ["source"],
include),
[
item2
for item1 in library_merge.get ("include", [])
for item2 in expand_parents (item1)
]))
excludes = set (map (
lambda exclude: "third-party/%s%s%s" % (
library_name,
library_merge ["source"],
exclude),
library_merge.get ("exclude", [])))
if includes or excludes:
source_index = (
git.IndexFile.from_tree (
self.git_repo,
"4b825dc642cb6eb9a060e54bf8d69288fbee4904"))
def predicate (item, depth):
return not excludes \
or item not in excludes
def prune (item, depth):
return (
includes
and item.path not in includes
) or (
excludes
and item.path in excludes
)
source_prune_length = len (
"third-party/%s%s/" % (
library_name,
library_merge ["source"]))
for item in source_tree.traverse (
predicate = lambda i, d: True,
prune = prune):
source_index.add (
[ git.Blob (
self.git_repo,
item.binsha,
item.mode,
item.path [source_prune_length : ]),
])
source_tree = (
source_index.write_tree ())
merged_index = (
git.IndexFile.from_tree (
self.git_repo,
unmerged_tree,
target_tree,
source_tree))
merged_tree = (
merged_index.write_tree ())
log.notice (
"MERGED TREE: " + unicode (merged_tree))
if os.path.exists (
"%s/%s" % (
self.project_path,
target_path)):
self.git_repo.git.rm (
"--force",
"-r",
target_path)
self.git_repo.git.read_tree (
unicode (merged_tree),
"--prefix",
target_path)
self.git_repo.git.read_tree (
unicode (source_tree),
"--prefix",
unmerged_path)
log.notice (
"INDEX TREE: " + unicode (
self.git_repo.index.write_tree ()))
if len (self.git_repo.index.diff ()):
self.git_repo.index.commit (
"Auto-merge changes from %s" % (
library_name))
num_merged += 1
except subprocess.CalledProcessError as error:
log.notice (
"Merge failed!")
log.output (
error.output)
num_failed += 1
if num_failed:
log.notice (
"Merged %s libraries with %s failures" % (
num_merged,
num_failed))
elif num_merged:
log.notice (
"Merged %s libraries" % (
num_merged))
def run(self, src_path, dst_path):
src = git.Repo(src_path)
dst = git.Repo.init(dst_path)
children = defaultdict(set) # binsha => set(binsha)
threads = list() # [commit]
depend = dict() # binsha => set(binsha)
for head in src.heads:
st = [head.commit]
while st:
commit = st.pop()
if commit.binsha not in depend:
depend[commit.binsha] = {parent.binsha for parent in commit.parents}
if not commit.parents:
threads.append(commit)
for c in commit.parents:
if c.binsha not in children:
st.append(c)
children[c.binsha].add(commit)
blob_map_cache = dict()
commit_binsha_map = dict() # old binsha => new binsha
height = dict() # new binsha => height
while threads:
commit = threads.pop()
index = dst.index
blobs = set()
for item in commit.tree.traverse():
key = item.binsha, item.mode, item.path
if item.type == 'blob':
if key in blob_map_cache:
if blob_map_cache[key] is not None:
value = blob_map_cache[key]
blobs.add(value)
else:
res = self.blob_map(item.data_stream, item.mode, item.path)
if res is not None:
data, mode, path = res
istream = dst.odb.store(IStream('blob', len(data), io.BytesIO(data)))
value = blob_map_cache[key] = istream.binsha, mode, path
blobs.add(value)
else:
blob_map_cache[key] = None
for data, mode, path in self.commit_add(commit):
istream = dst.odb.store(IStream('blob', len(data), io.BytesIO(data)))
blobs.add((istream.binsha, mode, path))
# remove/add only the differene
old_blobs = {(blob[1].binsha, blob[1].mode, blob[1].path) for blob in index.iter_blobs()}
to_remove = list(old_blobs - blobs)
to_add = list(blobs - old_blobs)
for i in range(0, len(to_remove), 128):
index.remove([git.Blob(dst, *t) for t in to_remove[i:i + 128]])
for i in range(0, len(to_add), 128):
index.add([git.Blob(dst, *t) for t in to_add[i:i + 128]])
parent_commits=[commit_binsha_map[parent.binsha] for parent in commit.parents]
message, author, authored_date, author_tz_offset, committer, committed_date, committer_tz_offset = self.commit_map(commit, commit.message, commit.author, commit.authored_date, commit.author_tz_offset, commit.committer, commit.committed_date, commit.committer_tz_offset)
author_date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(authored_date)) + ' ' + git.objects.util.altz_to_utctz_str(author_tz_offset)
commit_date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(committed_date)) + ' ' + git.objects.util.altz_to_utctz_str(committer_tz_offset)
skip_flag = False
if self.remove_empty_commits:
# detect grandparents
min_height = min(height[parent.binsha] for parent in parent_commits) if parent_commits else 0
st = parent_commits[:]
grandparents = set()
while st:
current = st.pop()
for grandparent in current.parents:
if grandparent.binsha not in grandparents:
grandparents.add(grandparent.binsha)
if height[grandparent.binsha] > min_height:
st.append(grandparent)
parent_commits = [parent for parent in parent_commits if parent.binsha not in grandparents]
# detect same parents
for i in range(len(parent_commits) - 1, -1, -1):
if parent_commits[i].binsha in set(parent.binsha for parent in parent_commits[:i]):
parent_commits.pop(i)
# skip empty commits
for parent in parent_commits:
if not index.diff(parent):
dst_commit = parent
skip_flag = True
break
if not skip_flag:
dst_commit = index.commit(message, parent_commits=parent_commits, author=author, committer=committer, author_date=author_date, commit_date=commit_date)
commit_binsha_map[commit.binsha] = dst_commit
height[dst_commit.binsha] = max(height[parent.binsha] for parent in dst_commit.parents) + 1 if dst_commit.parents else 0
self.progress(commit, dst_commit)
for child in children[commit.binsha]:
depend[child.binsha].remove(commit.binsha)
if not depend[child.binsha]:
threads.append(child)
for head in src.heads:
if not head.name in dst.heads:
dst.create_head(head.name)
dst.heads[head.name].commit = commit_binsha_map[head.commit.binsha]
def from_note(klass, repo, note):
blob = git.Blob(repo.repo,
binascii.a2b_hex(note['blob']),
path=note['path'])
return klass(repo, blob)
