def git_branches_differ(branch_a: str, branch_b: str, metadata_path: str) -> bool:
# Check to see if any files besides synth.metadata were added, modified, deleted.
diff_cmd = ["git", "diff", f"{branch_a}..{branch_b}"]
diff_cmd.extend(["--", ".", f":(exclude){metadata_path}"])
proc = executor.run(diff_cmd, stdout=subprocess.PIPE, universal_newlines=True)
proc.check_returncode()
if bool(proc.stdout):
return True
# Check to see if synth.metadata was added.
proc = executor.run(
["git", "diff", f"{branch_a}..{branch_b}", "--", metadata_path],
stdout=subprocess.PIPE,
universal_newlines=True,
)
proc.check_returncode()
diff_text = proc.stdout
pattern = "^--- /dev/null"
return bool(re.search(pattern, diff_text, re.MULTILINE))
def get_last_commit_to_file(file_path: str) -> str:
"""Returns the commit hash of the most recent change to a file."""
parent_dir = pathlib.Path(file_path).parent
proc = executor.run(
["git", "log", "--pretty=format:%H", "-1", "--no-decorate", file_path],
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=parent_dir,
)
proc.check_returncode()
return proc.stdout.strip()
def get_timestamp(self) -> datetime.datetime:
if self.timestamp is None:
unix_timestamp = executor.run(
["git", "log", "-1", "--pretty=%at", self.sha],
cwd=self.repo_path,
universal_newlines=True,
check=True,
stdout=subprocess.PIPE,
).stdout.strip()
self.timestamp = datetime.datetime.fromtimestamp(float(unix_timestamp))
return self.timestamp
def get_comment(self) -> str:
# Construct a comment using the text of the git commit.
if self.comment is None:
pretty = "--pretty=%B%n%nSource-Author: %an <%ae>%nSource-Date: %ad"
git_log: str = executor.run(
["git", "log", self.sha, "-1", "--no-decorate", pretty],
cwd=self.repo_path,
stdout=subprocess.PIPE,
universal_newlines=True,
check=True,
).stdout.strip()
self.comment = _compose_comment(self.remote, self.sha, git_log)
return self.comment
def enumerate_versions_for_working_repo(
metadata_path: str, sources: typing.List[typing.Dict[str, typing.Dict]]
) -> typing.List[autosynth.abstract_source.AbstractSourceVersion]:
"""Enumerates every commit after the most recent commit to metadata_path.
Special case for enumerating the change history of the repo that
we're actually generating.
Arguments:
metadata_path {str} -- Metadata file path.
Returns:
typing.List[autosynth.abstract_source.AbstractSourceVersion] -- versions
"""
# Get the repo root directory that contains metadata_path.
local_repo_dir = git.get_repo_root_dir(metadata_path)
# Find the most recent commit hash.
head_sha = executor.run(
["git", "log", "-1", "--pretty=%H"],
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=local_repo_dir,
check=True,
).stdout.strip()
# Get the remote url.
remote = executor.run(
["git", "remote", "get-url", "origin"],
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=local_repo_dir,
check=True,
).stdout.strip()
desc = f"This git repo ({remote})"
version = GitSourceVersion(local_repo_dir, head_sha, remote, desc, "self")
# The change from this repository must always be built first.
version.timestamp = datetime.datetime.fromtimestamp(0)
return [version]
def _execute(command: typing.List[str], env: typing.Any,
log_file_path: pathlib.Path) -> typing.Tuple[int, bytes]:
"""Helper to wrap command invocation for testing"""
# Ensure the logfile directory exists
log_file_path.parent.mkdir(parents=True, exist_ok=True)
with open(log_file_path, "wb+") as log_file:
result = executor.run(
command=command,
stdout=log_file,
stderr=subprocess.STDOUT,
check=False,
encoding="utf-8",
env=env,
)
with open(log_file_path, "rb") as fp:
return (result.returncode, fp.read())
def synthesize(self,
log_file_path: pathlib.Path,
environ: typing.Mapping[str, str] = None) -> str:
"""
Returns:
The log of the call to synthtool.
"""
logger.info("Running synthtool")
if not self.deprecated_execution:
command = [
sys.executable,
"-m",
"synthtool",
"--metadata",
self.metadata_path,
self.synth_py_path,
"--",
]
else:
# Execute the synthesis script directly (deprecated)
command = [sys.executable, self.synth_py_path]
logger.info(command)
logger.debug(f"log_file_path: {log_file_path}")
# Ensure the logfile directory exists
log_file_path.parent.mkdir(parents=True, exist_ok=True)
# Tee the output into a provided location so we can see the return the final output
tee_proc = subprocess.Popen(["tee", log_file_path],
stdin=subprocess.PIPE)
# Invoke synth.py.
synth_proc = executor.run(
command + self.extra_args,
stderr=subprocess.STDOUT,
stdout=tee_proc.stdin,
env=(environ or os.environ),
universal_newlines=True,
)
if synth_proc.returncode:
logger.error("Synthesis failed")
synth_proc.check_returncode() # Raise an exception.
with open(log_file_path, "rt") as fp:
return fp.read()
def get_commit_subject(repo_dir: str = None, sha: str = None) -> str:
"""Gets the subject line of the a commit.
Keyword Arguments:
repo_dir {str} -- a directory in the repo; None means use cwd. (default: {None})
sha {str} -- the sha of the commit. None means the most recent commit.
Returns:
{str} -- the subject line
"""
commit_message: str = executor.run(
["git", "log", "-1", "--no-decorate", "--format=%B"] + ([sha] if sha else []),
stdout=subprocess.PIPE,
universal_newlines=True,
check=True,
cwd=repo_dir,
).stdout
lines = commit_message.splitlines()
return lines[0].strip() if lines else ""
def get_repo_root_dir(repo_path: str) -> str:
"""Given a path to a file or dir in a repo, find the root directory of the repo.
Arguments:
repo_path {str} -- Any path into the repo.
Returns:
str -- The repo's root directory.
"""
path = pathlib.Path(repo_path)
if not path.is_dir():
path = path.parent
proc = executor.run(
["git", "rev-parse", "--show-toplevel"],
stdout=subprocess.PIPE,
universal_newlines=True,
cwd=str(path),
)
proc.check_returncode()
return proc.stdout.strip()
def _collect_trailers(commit_count: int, git_dir: typing.Optional[str] = None) -> str:
"""Collects the trailers from recent commits in the repo.
Only collects the two trailers we're interested in.
Arguments:
commit_count {int} -- Number of commits to collect trailers from.
Keyword Arguments:
git_dir {typing.Optional[str]} -- directory of git repo (default: {None})
Returns:
str -- The trailer lines from the recent commits.
"""
text = executor.run(
["git", "log", f"-{commit_count}", "--pretty=%b"],
universal_newlines=True,
check=True,
stdout=subprocess.PIPE,
cwd=git_dir,
).stdout
return _parse_trailers(text)
def get_commit_shas_since(sha: str, dir: str) -> typing.List[str]:
"""Gets the list of shas for commits committed after the given sha.
Arguments:
sha {str} -- The sha in the git history.
dir {str} -- An absolute path to a directory in the git repository.
Returns:
typing.List[str] -- A list of shas. The 0th sha is sha argument (the oldest sha).
"""
proc = executor.run(
["git", "log", f"{sha}..HEAD", "--pretty=%H", "--no-decorate"],
universal_newlines=True,
stdout=subprocess.PIPE,
cwd=dir,
)
proc.check_returncode()
shas = proc.stdout.split()
shas.append(sha)
shas.reverse()
return shas
def _get_pretty(self, pretty: str) -> str:
"""Gets the pretty log for this commit.
Arguments:
pretty {str} -- the git log pretty format
Returns:
str -- the output of the git log command, stripped of leading and trailing
whitespace
"""
git_log: str = executor.run(
[
"git", "log", self.sha, "-1", "--no-decorate",
f"--pretty={pretty}"
],
cwd=self.repo_path,
stdout=subprocess.PIPE,
universal_newlines=True,
check=True,
).stdout.strip()
return git_log
def commit_all_changes(message: str) -> int:
"""Commits all changes in the repo.
Args:
message (str): the commit message
Returns:
int: 1 if a change was commited, otherwise 0.
"""
executor.check_call(["git", "add", "-A"])
status = executor.run(
["git", "status", "--porcelain"],
universal_newlines=True,
stdout=subprocess.PIPE,
check=True,
).stdout.strip()
if status:
executor.check_call(["git", "commit", "-m", message])
return 1
else:
# There are no changes to commit.
return 0
def _inner_main(temp_dir: str) -> int:
"""
Returns:
int -- Number of commits committed to the repo.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--github-user", default=os.environ.get("GITHUB_USER"))
parser.add_argument("--github-email",
default=os.environ.get("GITHUB_EMAIL"))
parser.add_argument("--github-token",
default=os.environ.get("GITHUB_TOKEN"))
parser.add_argument("--repository",
default=os.environ.get("REPOSITORY"),
required=True)
parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH"))
parser.add_argument("--metadata-path",
default=os.environ.get("METADATA_PATH"))
parser.add_argument(
"--deprecated-execution",
default=False,
action="store_true",
help=
"If specified, execute synth.py directly instead of synthtool. This behavior is deprecated.",
)
parser.add_argument("--branch-suffix",
default=os.environ.get("BRANCH_SUFFIX", None))
parser.add_argument("--pr-title", default="")
parser.add_argument("extra_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
gh = github.GitHub(args.github_token)
branch = "-".join(filter(None, ["autosynth", args.branch_suffix]))
pr_title = args.pr_title or (
f"[CHANGE ME] Re-generated {args.synth_path or ''} to pick up changes in "
f"the API or client library generator.")
change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh,
branch)
# capture logs for later
base_synth_log_path = pathlib.Path(
os.path.realpath("./logs")) / args.repository
if args.synth_path:
base_synth_log_path /= args.synth_path
logger.info(f"logs will be written to: {base_synth_log_path}")
working_repo_path = synthtool_git.clone(
f"https://github.com/{args.repository}.git")
try:
os.chdir(working_repo_path)
git.configure_git(args.github_user, args.github_email)
git.setup_branch(branch)
if args.synth_path:
os.chdir(args.synth_path)
metadata_path = os.path.join(args.metadata_path or "",
"synth.metadata")
flags = autosynth.flags.parse_flags()
# Override flags specified in synth.py with flags specified in environment vars.
for key in flags.keys():
env_value = os.environ.get(key, "")
if env_value:
flags[key] = False if env_value.lower(
) == "false" else env_value
metadata = load_metadata(metadata_path)
multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS]
multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS]
if (not multiple_commits and not multiple_prs) or not metadata:
if change_pusher.check_if_pr_already_exists(branch):
return 0
synth_log = Synthesizer(
metadata_path,
args.extra_args,
deprecated_execution=args.deprecated_execution,
).synthesize(base_synth_log_path)
if not has_changes():
logger.info("No changes. :)")
sys.exit(EXIT_CODE_SKIPPED)
git.commit_all_changes(pr_title)
change_pusher.push_changes(1, branch, pr_title, synth_log)
return 1
else:
if not multiple_prs and change_pusher.check_if_pr_already_exists(
branch):
return 0 # There's already an existing PR
# Enumerate the versions to loop over.
sources = metadata.get("sources", [])
source_versions = [
git_source.enumerate_versions_for_working_repo(
metadata_path, sources)
]
# Add supported source version types below:
source_versions.extend(
git_source.enumerate_versions(sources, pathlib.Path(temp_dir)))
# Prepare to call synthesize loop.
synthesizer = Synthesizer(
metadata_path,
args.extra_args,
args.deprecated_execution,
"synth.py",
)
x = SynthesizeLoopToolbox(
source_versions,
branch,
temp_dir,
metadata_path,
args.synth_path,
base_synth_log_path,
)
if not multiple_commits:
change_pusher = SquashingChangePusher(change_pusher)
# Call the loop.
commit_count = synthesize_loop(x, multiple_prs, change_pusher,
synthesizer)
if commit_count == 0:
logger.info("No changes. :)")
sys.exit(EXIT_CODE_SKIPPED)
return commit_count
finally:
if args.synth_path:
# We're generating code in a mono repo. The state left behind will
# probably be useful for generating the next API.
pass
else:
# We're generating a single API in a single repo, and using a different
# repo to generate the next API. So the next synth will not be able to
# use any of this state. Clean it up to avoid running out of disk space.
executor.run(["git", "clean", "-fdx"], cwd=working_repo_path)
