def Load(cls, compiler, context, output_stream):
filename = cls._GetPersistedFilename(context)
if os.path.isfile(filename):
try:
match = RegularExpression.TemplateStringToRegex(
cls.TEMPLATE).match(open(filename).read())
if match:
data = base64.b64decode(match.group("data"))
return pickle.loads(
bytearray(data)), os.stat(filename)[stat.ST_MTIME]
except:
output_stream.write(
"WARNING: Context information associated with the previous compilation appears to be corrupt; new data will be generated.\n"
)
return cls(compiler, context), 0
class GitSourceControlManagement(DistributedSourceControlManagement):
"""Specializations for Git [https://www.git-scm.com/]"""
# ----------------------------------------------------------------------
# |
# | Public Properties
# |
# ----------------------------------------------------------------------
Name = "Git"
DefaultBranch = "master"
Tip = "head"
WorkingDirectories = [
".git",
]
IgnoreFilename = ".gitignore"
DetachedHeadPseudoBranchName = "__DetachedHeadPseudoBranchName_{Index}_{BranchName}__"
_DetachedHeadPseudoBranchName_regex = RegularExpression.TemplateStringToRegex(
DetachedHeadPseudoBranchName)
# ----------------------------------------------------------------------
# |
# | Public Methods
# |
# ----------------------------------------------------------------------
@classmethod
def Execute(
cls,
repo_root,
command,
strip=False,
newline=False,
):
command = command.replace("git ", 'git -C "{}" '.format(repo_root))
if cls.Diagnostics:
sys.stdout.write("VERBOSE: {}\n".format(command))
result, content = Process.Execute(
command,
environment=os.environ,
)
if strip:
content = content.strip()
if newline and not content.endswith('\n'):
content += '\n'
return result, content
# ----------------------------------------------------------------------
@classmethod
def IsAvailable(cls):
is_available = getattr(cls, "_cached_is_available", None)
if is_available is None:
_, output = cls.Execute(os.getcwd(), "git")
is_available = "usage: git" in output
setattr(cls, "_cached_is_available", is_available)
return is_available
# ----------------------------------------------------------------------
_cached_is_active = set()
@classmethod
def IsActive(cls, repo_root):
for k in cls._cached_is_active:
if repo_root.startswith(k):
return k
try:
result = os.path.isdir(cls.GetRoot(repo_root))
# Note that we are only caching positive results. This is to ensure
# that we walk into repositories associated with subdirs when provided
# with a parent dir that isn't a repository itself (caching a failure
# on a parent dir would prevent traversal into its children).
if result:
cls._cached_is_active.add(repo_root)
except:
result = False
return result
# ----------------------------------------------------------------------
@classmethod
def Create(cls, output_dir):
if os.path.isdir(output_dir):
raise Exception(
"The directory '{}' already exists and will not be overwritten"
.format(output_dir))
FileSystem.MakeDirs(output_dir)
return cls.Execute(os.getcwd(), 'git init "{}"'.format(output_dir))
# ----------------------------------------------------------------------
@classmethod
def Clone(cls, uri, output_dir, branch=None):
if os.path.isdir(output_dir):
raise Exception(
"The directory '{}' already exists and will not be overwritten."
.format(output_dir))
clone_path, clone_name = os.path.split(output_dir)
FileSystem.MakeDirs(clone_path)
return cls.Execute(
clone_path, 'git clone{branch} "{uri}" "{name}"'.format(
branch=' --branch "{}"'.format(branch) if branch else '',
uri=uri,
name=clone_name,
))
# ----------------------------------------------------------------------
@classmethod
def GetUniqueName(cls, repo_root):
result, output = cls.Execute(repo_root, "git remote -v")
assert result == 0, (result, output)
regex = re.compile(r"origin\s+(?P<url>.+?)\s+\(fetch\)")
for line in output.split('\n'):
match = regex.match(line)
if match:
return match.group("url")
# If here, we didn't find anything. Most of the time, this
# is an indication that the repo is local (no remote); return
# the path.
return os.path.realpath(repo_root)
# ----------------------------------------------------------------------
@classmethod
def Who(cls, repo_root):
# ----------------------------------------------------------------------
def GetValue(name):
result, output = cls.Execute(repo_root,
"git config {}".format(name))
assert result == 0, (result, output)
return output.strip()
# ----------------------------------------------------------------------
return "{} <{}>".format(
GetValue("user.name"),
GetValue("user.email"),
)
# ----------------------------------------------------------------------
_cached_roots = set()
@classmethod
def GetRoot(cls, repo_dir):
for k in cls._cached_roots:
if repo_dir.startswith(k):
return k
result, output = cls.Execute(
repo_dir,
"git rev-parse --show-toplevel",
strip=True,
)
assert result == 0, (result, output)
cls._cached_roots.add(output)
return output
# ----------------------------------------------------------------------
@classmethod
def IsRoot(cls, repo_dir):
return os.path.isdir(os.path.join(repo_dir, cls.WorkingDirectories[0]))
# ----------------------------------------------------------------------
@classmethod
def Clean(cls, repo_root, no_prompt=False):
if not no_prompt and not cls.AreYouSurePrompt(
textwrap.dedent(
# <Wrong hanging indentation> pylint: disable = C0330
"""\
This operation will revert any working changes.
THIS INCLUDES THE FOLLOWING:
- Any working edits
- Any files that have been added
""")):
return 0, "<<Skipped>>"
commands = [
"git reset --hard",
"git clean -df",
]
return cls.Execute(repo_root, " && ".join(commands))
# ----------------------------------------------------------------------
_GetBranches_regex = re.compile(
r"^\*?\s*\[(origin/)?(?P<name>\S+?)\]\s+.+?")
@classmethod
def GetBranches(cls, repo_root):
result, output = cls.Execute(repo_root, "git show-branch --list --all")
assert result == 0, (result, output)
branches = set()
for line in output.split('\n'):
match = cls._GetBranches_regex.match(line)
if not match:
continue
branch = match.group("name")
if branch not in branches:
branches.add(branch)
yield branch
# ----------------------------------------------------------------------
_GetCurrentBranch_regex = re.compile(r"\s*\*\s+(?P<name>.+)")
@classmethod
def GetCurrentBranch(cls, repo_root):
result, output = cls.Execute(repo_root, "git branch --no-color")
assert result == 0, (result, output)
if output:
for line in output.split('\n'):
match = cls._GetCurrentBranch_regex.match(line)
if match:
return match.group("name")
return cls.DefaultBranch
# ----------------------------------------------------------------------
@classmethod
def GetMostRecentBranch(cls, repo_root):
result, output = cls.Execute(
repo_root,
'git for-each-ref --sort=-committerdate --format="%(refname)"',
)
assert result == 0, (result, output)
for line in output.split('\n'):
parts = line.split('/')
if parts[1] == "remotes" and parts[2] == "origin":
return parts[3]
assert False, output
return None
# ----------------------------------------------------------------------
@classmethod
def CreateBranch(cls, repo_root, branch_name):
return cls.Execute(
repo_root, 'git branch "{name}" && git checkout "{name}"'.format(
name=branch_name))
# ----------------------------------------------------------------------
@classmethod
def SetBranch(cls, repo_root, branch_name):
return cls.Execute(repo_root, 'git checkout "{}"'.format(branch_name))
# ----------------------------------------------------------------------
@classmethod
def HasUntrackedWorkingChanges(cls, repo_root):
result, output = cls.Execute(
repo_root, "git ls-files --others --exclude-standard")
assert result == 0, (result, output)
return bool(output)
# ----------------------------------------------------------------------
@classmethod
def HasWorkingChanges(cls, repo_root):
result, output = cls.Execute(repo_root,
"git --no-pager diff -name-only")
assert result == 0, (result, output)
return bool(output)
# ----------------------------------------------------------------------
@classmethod
def GetChangeInfo(cls, repo_root, change):
# Not the spaces to work around issues with git
template = " %aN <%ae> %n %cd %n %s"
result, output = cls.Execute(
repo_root, 'git --no-pager show -s --format="{}" "{}"'.format(
template, change))
assert result == 0, (result, output)
lines = output.split('\n')
assert len(lines) >= 3, (len(lines), output)
d = {
"user": lines[0].lstrip(),
"date": lines[1].lstrip(),
"summary": lines[2].lstrip(),
}
d["files"] = cls.GetChangedFiles(repo_root, change)
return d
# ----------------------------------------------------------------------
_Commit_regex = re.compile(r"(?P<username>.+?)\s+\<(?P<email>.+?)\>")
@classmethod
def Commit(cls, repo_root, description, username=None):
# Git is particular about username format; massage it into the right
# format if necessary.
if username:
match = cls._Commit_regex.match(username)
if not match:
username = "******".format(username)
return cls.Execute(
repo_root,
'git commit -a --allow-empty -m "{desc}"{user}'.format(
desc=description.replace('"', '\\"'),
user='******'.format(username) if username else '',
),
)
# ----------------------------------------------------------------------
@classmethod
def Update(cls, repo_root, update_arg=None):
branch = cls.GetCurrentBranch(repo_root)
commands = [
'git merge --ff-only "origin/{}"'.format(branch),
]
if update_arg is None:
pass
elif isinstance(update_arg, EmptyUpdateMergeArg):
pass
elif isinstance(update_arg, BranchUpdateMergeArg):
commands.insert(0, 'git checkout "{}"'.format(update_arg.Branch))
elif isinstance(update_arg, (
ChangeUpdateMergeArg,
DateUpdateMergeArg,
BranchAndDateUpdateMergeArg,
)):
revision = cls._UpdateMergeArgToString(repo_root, update_arg)
# Updating to a specific revision within Git is interesting, as one will find
# themselves in a "DETACHED HEAD" state. While this makes a lot of sense from
# a commit perspective, it doesn't make as much sense from a reading perspective
# (especially in scenarios where it is necessary to derive the branch name from the
# current state, as will be the case during Reset). To work around this, Update to
# a new branch that is cleverly named in a way that can be parsed by commands that
# need this sort of information.
existing_branch_names = set(cls.GetBranches(repo_root))
index = 0
while True:
potential_branch_name = cls.DetachedHeadPseudoBranchName.format(
Index=index,
BranchName=branch,
)
if potential_branch_name not in existing_branch_names:
break
index += 1
commands.append('git checkout {} -b "{}"'.format(
revision, potential_branch_name))
else:
assert False, type(update_arg)
return cls.Execute(repo_root, " && ".join(commands))
# ----------------------------------------------------------------------
@classmethod
def Merge(cls, repo_root, merge_arg):
return cls.Execute(
repo_root, 'git merge --no-commit --no-ff {}'.format(
cls._UpdateMergeArgToString(repo_root, merge_arg)))
# ----------------------------------------------------------------------
@classmethod
def GetChangesSinceLastMerge(cls,
repo_root,
dest_branch,
source_merge_arg=None):
# Git is really screwed up. After a 30 minute search, I couldn't find a way to
# specify a branch and beginning revision in a single command. Therefore, I am
# doing it manually.
source_branch = None
additional_filters = []
post_decorator_func = None
# ----------------------------------------------------------------------
def GetDateOperator(arg):
if arg is None or arg:
return "since"
return "until"
# ----------------------------------------------------------------------
if source_merge_arg is None or isinstance(source_merge_arg,
EmptyUpdateMergeArg):
source_branch = cls.GetCurrentBranch(repo_root)
elif isinstance(source_merge_arg, ChangeUpdateMergeArg):
source_branch = cls._GetBranchAssociatedWithChange(
source_merge_arg.Change)
# ----------------------------------------------------------------------
def AfterChangeDecorator(changes):
starting_index = None
for index, change in enumerate(changes):
if change == source_merge_arg.Change:
starting_index = index
break
if starting_index is None:
return []
return changes[starting_index:]
# ----------------------------------------------------------------------
post_decorator_func = AfterChangeDecorator
elif isinstance(source_merge_arg, DateUpdateMergeArg):
source_branch = cls.GetCurrentBranch(repo_root)
additional_filters.append('--{}="{}"'.format(
GetDateOperator(source_merge_arg.GreaterThan),
StringSerialization.SerializeItem(DateTimeTypeInfo(),
source_merge_arg.Date),
))
elif isinstance(source_merge_arg, BranchUpdateMergeArg):
source_branch = source_merge_arg.Branch
elif isinstance(source_merge_arg, BranchAndDateUpdateMergeArg):
source_branch = source_merge_arg.Branch
additional_filters.append('--{}="{}"'.format(
GetDateOperator(source_merge_arg.GreaterThan),
StringSerialization.SerializeItem(DateTimeTypeInfo(),
source_merge_arg.Date),
))
else:
assert False, type(source_merge_arg)
command_line = 'git --no-pager log "{source_branch}" --not "{dest_branch}" --format="%H" --no-merges{additional_filters}' \
.format( source_branch=source_branch,
dest_branch=dest_branch,
additional_filters='' if not additional_filters else " {}".format(' '.join(additional_filters)),
)
result, output = cls.Execute(repo_root, command_line)
assert result == 0, (result, output)
changes = [line.strip() for line in output.split('\n') if line.strip()]
if post_decorator_func:
changes = post_decorator_func(changes)
return changes
# ----------------------------------------------------------------------
_GetChangedFiles_regex = re.compile(
r'^(?:\S|\?\?)\s+"?(?P<filename>.+)"?$')
@classmethod
def GetChangedFiles(cls, repo_root, change_or_changes_or_none):
if not change_or_changes_or_none:
result, output = cls.Execute(repo_root, 'git status --short')
assert result == 0, (result, output)
filenames = []
for line in [
line.strip() for line in output.split('\n')
if line.strip()
]:
match = cls._GetChangedFiles_regex(line)
assert match, line
filenames.append(
os.path.join(repo_root, match.group("filename")))
return filenames
changes = change_or_changes_or_none if isinstance(
change_or_changes_or_none, list) else [
change_or_changes_or_none,
]
command_line_template = 'git diff-tree --no-commit-id --name-only -r {}'
filenames = set()
for change in changes:
change = cls._UpdateMergeArgToString(repo_root,
ChangeUpdateMergeArg(change))
command_line = command_line_template.format(change)
result, output = cls.Execute(repo_root, command_line)
assert result == 0, (result, output)
for line in [
line.strip() for line in output.split('\n')
if line.strip()
]:
filename = os.path.join(repo_root, line)
if filename not in filenames:
filenames.add(filename)
return sorted(filenames)
# ----------------------------------------------------------------------
@classmethod
def EnumBlameInfo(cls, repo_root, filename):
result, output = cls.Execute(repo_root,
'git blame -s "{}"'.format(filename))
if result != 0:
# Don't produce an error if we are looking at a file that has been renamed/removed.
if "No such file or directory" in output:
return
assert False, (result, output)
regex = re.compile(
r"^(?P<revision>\S+)\s+(?P<line_number>\d+)\)(?: (?P<line>.*))?$")
for line in output.split('\n'):
if not line:
continue
match = regex.match(line)
if not match:
# Don't produce an error on a failure to enumerate binary files
if line.endswith("binary file"):
return
assert False, line
yield match.group("revision"), match.group("line_number")
# ----------------------------------------------------------------------
@classmethod
def EnumTrackedFiles(cls, repo_root):
temp_filename = CurrentShell.CreateTempFilename()
result, output = cls.Execute(
repo_root, 'git ls-files > "{}"'.format(temp_filename))
assert result == 0, (result, output)
assert os.path.isfile(temp_filename), temp_filename
with CallOnExit(lambda: os.remove(temp_filename)):
with open(temp_filename) as f:
for line in f.readlines():
line = line.strip()
if line:
yield os.path.join(repo_root, line)
# ----------------------------------------------------------------------
@classmethod
def CreatePatch(
cls,
repo_root,
output_filename,
start_change=None,
end_change=None,
):
if not start_change or not end_change:
command_line = 'git diff -g > "{}"'.format(output_filename)
else:
command_line = 'git diff -g {start} {end} > "{filename}"'.format(
start=start_change,
end=end_change,
filename=output_filename,
)
return cls.Execute(repo_root, command_line)
# ----------------------------------------------------------------------
@classmethod
def ApplyPatch(
cls,
repo_root,
patch_filename,
commit=False,
):
if commit:
raise Exception(
"Git does not support applying a patch without committing")
return cls.Execute(repo_root, 'git apply "{}"'.format(patch_filename))
# ----------------------------------------------------------------------
@classmethod
def Reset(
cls,
repo_root,
no_prompt=False,
no_backup=False,
):
if not no_prompt and not cls.AreYouSurePrompt(
textwrap.dedent(
# <Wrong hanging indentation> pylint: disable = C0330
"""\
This operation will revert your local repository to match the state of the remote repository.
THIS INCLUDES THE FOLLOWING:
- Any working edits
- Any files that have been added
- Any committed changes that have not been pushed to the remote repository
""")):
return 0, ''
commands = []
# See if we are looking at a detached head pseudo branch. If so, extract the actual
# branch name and switch to that before running the other commands.
branch = cls.GetCurrentBranch(repo_root)
match = cls._DetachedHeadPseudoBranchName_regex.match(branch)
if match:
branch = match.group("BranchName")
commands.append('git checkout "{}"'.format(branch))
# Remove any of the pseudo branches that have been created
for potential_delete_branch in cls.GetBranches(repo_root):
if cls._DetachedHeadPseudoBranchName_regex.match(
potential_delete_branch):
commands.append(
'git branch -D "{}"'.format(potential_delete_branch))
commands += [
'git reset --hard "origin/{}"'.format(branch),
'git clean -df',
]
return cls.Execute(repo_root, ' && '.join(commands))
# ----------------------------------------------------------------------
@classmethod
def HasUpdateChanges(cls, repo_root):
result, output = cls.Execute(repo_root, "git status -uno")
assert result == 0, (result, output)
return "Your branch is behind" in output or "have diverged" in output
# ----------------------------------------------------------------------
@classmethod
def HasLocalChanges(cls, repo_root):
result, output = cls.Execute(
repo_root, 'git --no-pager log --branches --not --remotes')
assert result == 0 or (result == 1
and "no changes found" in output), (result,
output)
return bool(output)
# ----------------------------------------------------------------------
@classmethod
def GetLocalChanges(cls, repo_root):
result, output = cls.Execute(repo_root, 'git remote update')
assert result == 0, (result, output)
result, output = cls.Execute(
repo_root,
'git --no-pager log "origin/{}..HEAD" --format="%H"'.format(
cls.GetCurrentBranch(repo_root)))
assert result == 0, (result, output)
return [line.strip() for line in output.split('\n') if line.strip()]
# ----------------------------------------------------------------------
@classmethod
def HasRemoteChanges(cls, repo_root):
result, output = cls.Execute(repo_root, 'git remote update')
assert result == 0, (result, output)
return cls.HasUpdateChanges(repo_root)
# ----------------------------------------------------------------------
@classmethod
def GetRemoteChanges(cls, repo_root):
result, output = cls.Execute(repo_root, 'git remote update')
assert result == 0, (result, output)
result, output = cls.Execute(
repo_root,
'git --no-pager log "HEAD..origin/{}" --format="%H"'.format(
cls.GetCurrentBranch(repo_root)))
assert result == 0, (result, output)
return [line.strip() for line in output.split('\n') if line.strip()]
# ----------------------------------------------------------------------
@classmethod
def Push(cls, repo_root, create_remote_branch=False):
commands = [
'git push',
'git push --tags',
]
if create_remote_branch:
commands[0] += ' --set-upstream origin "{}"'.format(
cls.GetCurrentBranch(repo_root))
return cls.Execute(repo_root, " && ".join(commands))
# ----------------------------------------------------------------------
@classmethod
def Pull(cls, repo_root, branch_or_branches=None):
commands = []
if isinstance(branch_or_branches, six.string_types):
branch_or_branches = [
branch_or_branches,
]
existing_branches = set(cls.GetBranches(repo_root))
for branch_name in (branch_or_branches or []):
if branch_name not in existing_branches:
commands.append(
'git checkout -b "{name}" "origin/{name}"'.format(
name=branch_name))
commands += [
'git fetch --all',
'git fetch --all --tags',
]
return cls.Execute(repo_root, " && ".join(commands))
# ----------------------------------------------------------------------
# |
# | Private Methods
# |
# ----------------------------------------------------------------------
@classmethod
def _AddFilesImpl(cls, repo_root, filenames):
return cls.Execute(
repo_root, 'git add {}'.format(' '.join(
['"{}"'.format(filename) for filename in filenames])))
# ----------------------------------------------------------------------
@classmethod
def _UpdateMergeArgToString(cls, repo_root, update_arg):
# ----------------------------------------------------------------------
def NormalizeChange(change):
result, output = cls.Execute(
repo_root,
'git --no-pager log {} -n 1 --format="%H"'.format(change))
assert result == 0, (result, output)
output = output.strip()
if output.startswith("* "):
output = output[len("* "):]
return output
# ----------------------------------------------------------------------
def DateAndBranch(date, branch, operator):
assert date
if branch:
# ----------------------------------------------------------------------
def BranchGenerator():
yield branch
# ----------------------------------------------------------------------
else:
# ----------------------------------------------------------------------
def BranchGenerator():
for branch in [
cls.GetCurrentBranch(repo_root),
cls.DefaultBranch,
]:
yield branch
# ----------------------------------------------------------------------
assert BranchGenerator
if operator:
operator = "since"
else:
operator = "until"
for branch in BranchGenerator():
result, output = cls.Execute(
repo_root,
'git --no-pager long "--branches=*{}" "--{}={}" -n 1 --format="%H"'
.format(branch, operator, date))
if result == 0 and output:
return output
raise Exception("Revision not found")
# ----------------------------------------------------------------------
dispatch_map = {
EmptyUpdateMergeArg:
lambda: "",
ChangeUpdateMergeArg:
lambda: NormalizeChange(update_arg.Change),
DateUpdateMergeArg:
lambda: DateAndBranch(update_arg.Date, None, update_arg.GreaterThan
),
BranchUpdateMergeArg:
lambda: update_arg.Branch,
BranchAndDateUpdateMergeArg:
lambda: DateAndBranch(update_arg.Date, update_arg.Branch,
update_arg.GreaterThan),
}
assert type(update_arg) in dispatch_map, type(update_arg)
return dispatch_map[type(update_arg)]()
# ----------------------------------------------------------------------
@classmethod
def _GetBranchAssociatedWithChange(cls, repo_root, change):
result, output = cls.Execute(repo_root,
'git branch --contains {}'.format(change))
assert result == 0, (result, output)
output = output.strip()
if output.startswith("* "):
output = output[len("* "):]
return output
def FromBuildFile( cls,
build_filename,
strip_path=None,
):
assert os.path.isfile(build_filename), build_filename
# Extract metadata auto-generated from the build file to create the
# configuration. Note that some portions of the metadata will be based
# on the current dir, so switch to the root dir before extracting the
# information to ensure that things work as expected.
if strip_path:
assert os.path.isdir(strip_path), strip_path
current_dir = os.getcwd()
os.chdir(strip_path)
# ----------------------------------------------------------------------
def Cleanup():
os.chdir(current_dir)
# ----------------------------------------------------------------------
else:
# ----------------------------------------------------------------------
def Cleanup():
pass
# ----------------------------------------------------------------------
with CallOnExit(Cleanup):
result, output = Process.Execute('python "{}" Metadata'.format(build_filename))
assert result == 0, (result, output)
match = RegularExpression.TemplateStringToRegex(cls._VIEW_METADATA_TEMPLATE).match(output)
if not match:
raise Exception("'{}' did not produce valid metadata results".format(build_filename))
# ----------------------------------------------------------------------
def FromList(value):
value = match.group(value)
return [] if value == "None" else [ item.strip() for item in value.split(',') if item.strip() ]
# ----------------------------------------------------------------------
def FromOptional(value):
value = match.group(value)
return None if value == "None" else value
# ----------------------------------------------------------------------
def FromBool(value):
value = match.group(value)
return value == "True"
# ----------------------------------------------------------------------
return cls( exposed_functions=FromList("commands"),
config=Configuration( name=match.group("name"),
priority=int(match.group("priority")),
suggested_output_dir_location=match.group("suggested_output_dir_location"),
requires_output_dir=FromBool("requires_output_dir"),
configurations=FromList("configurations"),
required_development_environment=FromOptional("required_dev_environment"),
required_development_configurations=FromList("required_dev_configurations"),
disable_if_dependency_environment=FromBool("disable_if_dependency"),
),
)