def fork(self) -> typing.List["SynthesizeLoopToolbox"]:
"""Create a new toolbox for each source.
Each fork contains the same list of sources. In each fork, only one
source contains its full list of versions. The other sources contain
the oldest version only.
Returns:
[typing.List[SynthesizeLoopToolbox]] -- A toolbox for each source.
"""
forks = []
for i, _group in enumerate(self.version_groups):
new_groups = [[g[0]] for g in self.version_groups]
new_groups[i] = self.version_groups[i]
source_name = self.version_groups[i][0].get_source_name()
fork_branch = f"{self.branch}-{source_name}"
fork = SynthesizeLoopToolbox(
new_groups,
fork_branch,
self._temp_dir,
self._metadata_path,
self._synth_path,
self.log_dir_path / source_name,
)
fork.source_name = source_name
fork.commit_count = self.commit_count
fork.version_zero = self.version_zero
executor.check_call(["git", "branch", "-f", fork_branch])
forks.append(fork)
return forks
def configure_git(user: str, email: str) -> None:
with open(GLOBAL_GITIGNORE_FILE, "w") as fh:
fh.write(GLOBAL_GITIGNORE)
executor.check_call(
["git", "config", "--global", "core.excludesfile", GLOBAL_GITIGNORE_FILE]
)
executor.check_call(["git", "config", "user.name", user])
executor.check_call(["git", "config", "user.email", email])
executor.check_call(["git", "config", "push.default", "simple"])
def patch_merge(
branch_name: str, patch_file_path: str, git_repo_dir: str = None
) -> None:
"""Merges a branch via `git diff | git apply`.
Does not commit changes. Modifies files only.
Arguments:
branch_name {str} -- The other branch to merge into this one.
patch_file_path {str} -- The path where the patch file will be (over)written.
Keyword Arguments:
git_repo_dir {str} -- The repo directory (default: current working directory)
"""
with open(patch_file_path, "wb+") as patch_file:
executor.check_call(
["git", "diff", "HEAD", branch_name], stdout=patch_file, cwd=git_repo_dir
)
if os.stat(patch_file_path).st_size:
executor.check_call(["git", "apply", patch_file_path], cwd=git_repo_dir)
def synthesize_loop(
toolbox: SynthesizeLoopToolbox,
multiple_prs: bool,
change_pusher: AbstractChangePusher,
synthesizer: AbstractSynthesizer,
) -> int:
"""Loops through all source versions and creates a commit for every version
changed that caused a change in the generated code.
Arguments:
toolbox {SynthesizeLoopToolbox} -- a toolbox
multiple_prs {bool} -- True to create one pull request per source.
change_pusher {AbstractChangePusher} -- Used to push changes to github.
synthesizer {AbstractSynthesizer} -- Invokes synthesize.
Returns:
int -- Number of commits committed to this repo.
"""
if not toolbox.versions:
return 0 # No versions, nothing to synthesize.
try:
if multiple_prs:
commit_count = 0
for fork in toolbox.fork():
if change_pusher.check_if_pr_already_exists(fork.branch):
continue
executor.check_call(["git", "checkout", fork.branch])
synthesize_inner_loop(fork, synthesizer)
commit_count += fork.commit_count
if fork.source_name == "self" or fork.count_commits_with_context(
) > 0:
fork.push_changes(change_pusher)
return commit_count
except Exception as e:
logger.error(e)
pass # Fall back to non-forked loop below.
if change_pusher.check_if_pr_already_exists(toolbox.branch):
return 0
synthesize_inner_loop(toolbox, synthesizer)
toolbox.push_changes(change_pusher)
return toolbox.commit_count
def commit_all_changes(message: str) -> int:
"""Commits all changes in the repo.
Args:
message (str): the commit message
Returns:
int: 1 if a change was commited, otherwise 0.
"""
executor.check_call(["git", "add", "-A"])
status = executor.run(
["git", "status", "--porcelain"],
universal_newlines=True,
stdout=subprocess.PIPE,
check=True,
).stdout.strip()
if status:
executor.check_call(["git", "commit", "-m", message])
return 1
else:
# There are no changes to commit.
return 0
def synthesize_version_in_new_branch(self,
synthesizer: AbstractSynthesizer,
index: int) -> bool:
"""Invokes the synthesizer on the version specified by index.
Stores the result in a new branch.
Leaves the current branch unchanged.
Arguments:
synthesizer {AbstractSynthesizer} -- A synthesizer.
index {int} -- index into self.versions
Returns:
bool -- True if the code generated differs.
"""
# Did we already generate this version? Return cached result.
branch_already_has_changes = self.versions[index].branch_has_changes
if branch_already_has_changes is not None:
return branch_already_has_changes
self.apply_version(index)
self.checkout_new_branch(index)
try:
if 0 == index:
if self.version_zero.branch_name:
# Reuse version zero built for another source.
executor.check_call([
"git", "merge", "--ff-only",
self.version_zero.branch_name
])
return self.version_zero.has_changes
synth_log_path = self.log_dir_path / str(index) / "sponge_log.log"
if index + 1 == len(self.versions):
# The youngest version. Let exceptions raise because the
# current state is broken, and there's nothing we can do.
synthesizer.synthesize(synth_log_path, self.environ)
else:
synthesizer.synthesize_and_catch_exception(
synth_log_path, self.environ)
# Save changes into the sub branch.
i_has_changes = has_changes()
git.commit_all_changes(self.versions[index].version.get_comment())
if 0 == index:
# Record version zero info so other sources can reuse.
self.version_zero.branch_name = self.sub_branch(0)
self.version_zero.has_changes = i_has_changes
# Cache the outcome.
self.versions[index].branch_has_changes = i_has_changes
return i_has_changes
finally:
executor.check_call(["git", "reset", "--hard", "HEAD"])
executor.check_call(["git", "checkout", self.branch])
def push_changes(
self, commit_count: int, branch: str, pr_title: str, synth_log: str = ""
) -> AbstractPullRequest:
if commit_count < 2:
# Only one change, no need to squash.
return self.inner_change_pusher.push_changes(
commit_count, branch, pr_title, synth_log
)
executor.check_call(["git", "checkout", branch]) # Probably redundant.
with tempfile.NamedTemporaryFile() as message_file:
# Collect the commit messages into a temporary file.
message_file.write("changes triggered by multiple versions\n\n".encode())
message_file.flush()
executor.run(
["git", "log", f"-{commit_count}", "--format=* %s%n%b"],
stdout=message_file,
check=True,
)
message_file.file.close() # type: ignore
# Do a git dance to construct a branch with the commits squashed.
temp_branch = str(uuid.uuid4())
executor.check_call(["git", "branch", "-m", temp_branch])
executor.check_call(["git", "checkout", "master"])
executor.check_call(["git", "checkout", "-b", branch])
executor.check_call(["git", "merge", "--squash", temp_branch])
executor.check_call(["git", "commit", "-F", message_file.name])
return self.inner_change_pusher.push_changes(1, branch, pr_title, synth_log)
def checkout_sub_branch(self, index: int):
"""Check out the branch for the version."""
executor.check_call(["git", "checkout", self.sub_branch(index)])
def checkout_new_branch(self, index: int) -> None:
"""Create a new branch for the version."""
executor.check_call(["git", "branch", "-f", self.sub_branch(index)])
executor.check_call(["git", "checkout", self.sub_branch(index)])
def metadata_contains_generated_files(self, branch_name: str) -> bool:
executor.check_call(["git", "checkout", branch_name])
metadata = load_metadata(self._metadata_path)
return bool(metadata.get("generatedFiles"))
def synthesize_loop(
toolbox: SynthesizeLoopToolbox,
multiple_prs: bool,
change_pusher: AbstractChangePusher,
synthesizer: AbstractSynthesizer,
) -> int:
"""Loops through all source versions and creates a commit for every version
changed that caused a change in the generated code.
Arguments:
toolbox {SynthesizeLoopToolbox} -- a toolbox
multiple_prs {bool} -- True to create one pull request per source.
change_pusher {AbstractChangePusher} -- Used to push changes to github.
synthesizer {AbstractSynthesizer} -- Invokes synthesize.
Returns:
int -- Number of commits committed to this repo.
"""
if not toolbox.versions:
return 0 # No versions, nothing to synthesize.
# Synthesize the library with the most recent versions of all sources.
youngest = len(toolbox.versions) - 1
has_changes = toolbox.synthesize_version_in_new_branch(
synthesizer, youngest)
if not has_changes:
if (not toolbox.metadata_contains_generated_files(toolbox.branch)
and toolbox.metadata_contains_generated_files(
toolbox.sub_branch(youngest)) and
not change_pusher.check_if_pr_already_exists(toolbox.branch)):
# Special case: the repo owner turned on obsolete file tracking.
# Generate a one-time PR containing only metadata changes.
executor.check_call(["git", "checkout", toolbox.branch])
executor.check_call(
["git", "merge", "--squash",
toolbox.sub_branch(youngest)])
pr_title = "chore: start tracking obsolete files"
executor.check_call(["git", "commit", "-m", pr_title])
pr = change_pusher.push_changes(1, toolbox.branch, pr_title)
pr.add_labels(["context: full"])
return 1
return 0 # No changes, nothing to do.
try:
if multiple_prs:
commit_count = 0
for fork in toolbox.fork():
if change_pusher.check_if_pr_already_exists(fork.branch):
continue
executor.check_call(["git", "checkout", fork.branch])
synthesize_inner_loop(fork, synthesizer)
commit_count += fork.commit_count
if fork.source_name == "self" or fork.count_commits_with_context(
) > 0:
fork.push_changes(change_pusher)
return commit_count
except Exception as e:
logger.error(e)
# Fallback to the single_pr loop to try to make some progress.
synthesize_loop_single_pr(toolbox, change_pusher, synthesizer)
# But still report the failure.
raise
return synthesize_loop_single_pr(toolbox, change_pusher, synthesizer)
def push_changes(branch):
executor.check_call(["git", "push", "--force", "origin", branch])
def commit_all_changes(message):
executor.check_call(["git", "add", "-A"])
executor.check_call(["git", "commit", "-m", message])
def setup_branch(branch: str) -> None:
executor.check_call(["git", "branch", "-f", branch])
executor.check_call(["git", "checkout", branch])
