def synthesize_library(
library: typing.Dict,
github_token: str,
extra_args: typing.List[str],
base_log_path: pathlib.Path,
runner: Runner = _execute,
) -> typing.Dict:
"""Run autosynth on a single library.
Arguments:
library {dict} - Library configuration
"""
logger.info(f"Synthesizing {library['name']}.")
command = [sys.executable, "-m", "autosynth.synth"]
env = os.environ
env["GITHUB_TOKEN"] = github_token
library_args = [
"--repository",
library["repository"],
"--synth-path",
library.get("synth-path", ""),
"--branch-suffix",
library.get("branch-suffix", ""),
"--pr-title",
library.get("pr-title", ""),
]
if library.get("metadata-path"):
library_args.extend(["--metadata-path", library.get("metadata-path")])
if library.get("deprecated-execution", False):
library_args.append("--deprecated-execution")
log_file_path = base_log_path / library["repository"] / "sponge_log.log"
# run autosynth in a separate process
(returncode, output) = runner(
command + library_args + library.get("args", []) + extra_args,
env,
log_file_path,
)
error = returncode not in (0, synth.EXIT_CODE_SKIPPED)
skipped = returncode == synth.EXIT_CODE_SKIPPED
if error:
logger.error(f"Synthesis failed for {library['name']}")
return {
"name": library["name"],
"output": output,
"error": error,
"skipped": skipped,
}
def check_if_pr_already_exists(self, branch) -> bool:
repo = self._repository
owner = repo.split("/")[0]
prs = self._gh.list_pull_requests(repo,
state="open",
head=f"{owner}:{branch}")
if prs:
pr = prs[0]
logger.info(f'PR already exists: {pr["html_url"]}')
return bool(prs)
def _file_or_comment_on_issue(gh, name, repository, issue_title,
existing_issue, output):
# GitHub rejects issues with bodies > 64k
output_to_report = output[-10000:]
sponge_log_url = _get_sponge_log_url(repository)
message = f"""\
Please investigate and fix this issue within 5 business days. While it remains broken,
this library cannot be updated with changes to the {name} API, and the library grows
stale.
See https://github.com/googleapis/synthtool/blob/master/autosynth/TroubleShooting.md
for trouble shooting tips.
Here's the output from running `synth.py`:
```
{output_to_report}
```
Google internal developers can see the full log [here]({sponge_log_url}).
"""
if not existing_issue:
issue_details = (
f"Hello! Autosynth couldn't regenerate {name}. :broken_heart:\n\n{message}"
)
labels = ["autosynth failure", "priority: p1", "type: bug"]
api_label = gh.get_api_label(repository, name)
if api_label:
labels.append(api_label)
issue = gh.create_issue(
repository,
title=issue_title,
body=issue_details,
labels=labels,
)
logger.info(f"Opened issue: {issue['url']}")
# otherwise leave a comment on the existing issue.
else:
comment_body = (
f"Autosynth is still having trouble generating {name}. :sob:\n\n{message}"
)
gh.create_issue_comment(
repository,
issue_number=existing_issue["number"],
comment=comment_body,
)
logger.info(f"Updated issue: {existing_issue['url']}")
def _close_issue(gh, repository: str, existing_issue: dict):
if existing_issue is None:
return
logger.info(f"Closing issue: {existing_issue['url']}")
gh.create_issue_comment(
repository,
issue_number=existing_issue["number"],
comment="Autosynth passed, closing! :green_heart:",
)
gh.patch_issue(
repository, issue_number=existing_issue["number"], state="closed",
)
def check_if_pr_already_exists(self, branch) -> bool:
repo = self._repository
owner = repo.split("/")[0]
prs = self._gh.list_pull_requests(repo, state="open", head=f"{owner}:{branch}")
if prs:
pr = prs[0]
self._existing_pull_requests[branch] = pr
logger.info(f'PR already exists: {pr["html_url"]}')
body: str = pr["body"]
if REGENERATE_CHECKBOX_TEXT in body:
logger.info("Someone requested the PR to be regenerated.")
return False
return bool(prs)
def _file_or_comment_on_issue(gh, name, repository, issue_title,
existing_issue, output):
# GitHub rejects issues with bodies > 64k
output_to_report = output[-10000:]
sponge_log_url = _get_sponge_log_url(repository)
message = f"""\
Here's the output from running `synth.py`:
```
{output_to_report}
```
Google internal developers can see the full log [here]({sponge_log_url}).
"""
if not existing_issue:
issue_details = (
f"Hello! Autosynth couldn't regenerate {name}. :broken_heart:\n\n{message}"
)
labels = ["autosynth failure", "priority: p1", "type: bug"]
api_label = gh.get_api_label(repository, name)
if api_label:
labels.append(api_label)
issue = gh.create_issue(
repository,
title=issue_title,
body=issue_details,
labels=labels,
)
logger.info(f"Opened issue: {issue['url']}")
# otherwise leave a comment on the existing issue.
else:
comment_body = (
f"Autosynth is still having trouble generating {name}. :sob:\n\n{message}"
)
gh.create_issue_comment(
repository,
issue_number=existing_issue["number"],
comment=comment_body,
)
logger.info(f"Updated issue: {existing_issue['url']}")
def synthesize(self,
log_file_path: pathlib.Path,
environ: typing.Mapping[str, str] = None) -> str:
"""
Returns:
The log of the call to synthtool.
"""
logger.info("Running synthtool")
if not self.deprecated_execution:
command = [
sys.executable,
"-m",
"synthtool",
"--metadata",
self.metadata_path,
self.synth_py_path,
"--",
]
else:
# Execute the synthesis script directly (deprecated)
command = [sys.executable, self.synth_py_path]
logger.info(command)
logger.debug(f"log_file_path: {log_file_path}")
# Ensure the logfile directory exists
log_file_path.parent.mkdir(parents=True, exist_ok=True)
# Tee the output into a provided location so we can see the return the final output
tee_proc = subprocess.Popen(["tee", log_file_path],
stdin=subprocess.PIPE)
# Invoke synth.py.
synth_proc = executor.run(
command + self.extra_args,
stderr=subprocess.STDOUT,
stdout=tee_proc.stdin,
env=(environ or os.environ),
universal_newlines=True,
)
if synth_proc.returncode:
logger.error("Synthesis failed")
synth_proc.check_returncode() # Raise an exception.
with open(log_file_path, "rt") as fp:
return fp.read()
def has_changes():
output = subprocess.check_output(["git", "status", "--porcelain"])
output = output.decode("utf-8").strip()
logger.info("Changed files:")
logger.info(output)
# Parse the git status output. Ignore any blank lines.
changed_files = [line.strip() for line in output.split("\n") if line]
# Ignore any files that are in our IGNORED_FILES set and only report
# that there are changes if these ignored files are not the *only* changed
# files.
filtered_changes = []
for file in changed_files:
for expr in IGNORED_FILE_PATTERNS:
if expr.match(file):
break
else:
filtered_changes.append(file)
return True if filtered_changes else False
def synthesize_library(
library: typing.Dict,
github_token: str,
extra_args: typing.List[str],
base_log_path: pathlib.Path,
runner: Runner = _execute,
) -> typing.Dict:
"""Run autosynth on a single library.
Arguments:
library {dict} - Library configuration
"""
logger.info(f"Synthesizing {library['name']}.")
command = [sys.executable, "-m", "autosynth.synth"]
env = os.environ
env["GITHUB_TOKEN"] = github_token
library_args = [
"--repository",
library["repository"],
"--synth-path",
library.get("synth-path", ""),
"--branch-suffix",
library.get("branch-suffix", ""),
"--pr-title",
library.get("pr-title", ""),
"--base-log-dir",
str(base_log_path),
]
if library.get("metadata-path"):
library_args.extend(["--metadata-path", library.get("metadata-path")])
if library.get("deprecated-execution", False):
library_args.append("--deprecated-execution")
log_file_dir = (pathlib.Path(base_log_path) / pathlib.Path(
library.get("synth-path", "") or library["repository"]).name)
log_file_path = log_file_dir / "sponge_log.log"
# run autosynth in a separate process
(returncode, output) = runner(
command + library_args + library.get("args", []) + extra_args,
env,
log_file_path,
)
error = returncode not in (0, synth.EXIT_CODE_SKIPPED)
skipped = returncode == synth.EXIT_CODE_SKIPPED
# Leave a sponge_log.xml side-by-side with sponge_log.log, and sponge
# will understand they're for the same task and render them accordingly.
results = [{
"name": library["name"],
"error": error,
"output": "See the test log.",
"skipped": skipped,
}]
make_report(library["name"], results, log_file_dir)
if error:
logger.error(f"Synthesis failed for {library['name']}")
return {
"name": library["name"],
"output": output.decode("utf-8"),
"error": error,
"skipped": skipped,
}
def _inner_main(temp_dir: str) -> int:
"""
Returns:
int -- Number of commits committed to the repo.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--github-user", default=os.environ.get("GITHUB_USER"))
parser.add_argument("--github-email",
default=os.environ.get("GITHUB_EMAIL"))
parser.add_argument("--github-token",
default=os.environ.get("GITHUB_TOKEN"))
parser.add_argument("--repository",
default=os.environ.get("REPOSITORY"),
required=True)
parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH"))
parser.add_argument("--metadata-path",
default=os.environ.get("METADATA_PATH"))
parser.add_argument(
"--deprecated-execution",
default=False,
action="store_true",
help=
"If specified, execute synth.py directly instead of synthtool. This behavior is deprecated.",
)
parser.add_argument("--branch-suffix",
default=os.environ.get("BRANCH_SUFFIX", None))
parser.add_argument("--pr-title", default="")
parser.add_argument("extra_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
gh = github.GitHub(args.github_token)
branch = "-".join(filter(None, ["autosynth", args.branch_suffix]))
pr_title = args.pr_title or (
f"[CHANGE ME] Re-generated {args.synth_path or ''} to pick up changes in "
f"the API or client library generator.")
change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh,
branch)
# capture logs for later
base_synth_log_path = pathlib.Path(
os.path.realpath("./logs")) / args.repository
if args.synth_path:
base_synth_log_path /= args.synth_path
logger.info(f"logs will be written to: {base_synth_log_path}")
working_repo_path = synthtool_git.clone(
f"https://github.com/{args.repository}.git")
try:
os.chdir(working_repo_path)
git.configure_git(args.github_user, args.github_email)
git.setup_branch(branch)
if args.synth_path:
os.chdir(args.synth_path)
metadata_path = os.path.join(args.metadata_path or "",
"synth.metadata")
flags = autosynth.flags.parse_flags()
# Override flags specified in synth.py with flags specified in environment vars.
for key in flags.keys():
env_value = os.environ.get(key, "")
if env_value:
flags[key] = False if env_value.lower(
) == "false" else env_value
metadata = load_metadata(metadata_path)
multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS]
multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS]
if (not multiple_commits and not multiple_prs) or not metadata:
if change_pusher.check_if_pr_already_exists(branch):
return 0
synth_log = Synthesizer(
metadata_path,
args.extra_args,
deprecated_execution=args.deprecated_execution,
).synthesize(base_synth_log_path)
if not has_changes():
logger.info("No changes. :)")
sys.exit(EXIT_CODE_SKIPPED)
git.commit_all_changes(pr_title)
change_pusher.push_changes(1, branch, pr_title, synth_log)
return 1
else:
if not multiple_prs and change_pusher.check_if_pr_already_exists(
branch):
return 0 # There's already an existing PR
# Enumerate the versions to loop over.
sources = metadata.get("sources", [])
source_versions = [
git_source.enumerate_versions_for_working_repo(
metadata_path, sources)
]
# Add supported source version types below:
source_versions.extend(
git_source.enumerate_versions(sources, pathlib.Path(temp_dir)))
# Prepare to call synthesize loop.
synthesizer = Synthesizer(
metadata_path,
args.extra_args,
args.deprecated_execution,
"synth.py",
)
x = SynthesizeLoopToolbox(
source_versions,
branch,
temp_dir,
metadata_path,
args.synth_path,
base_synth_log_path,
)
if not multiple_commits:
change_pusher = SquashingChangePusher(change_pusher)
# Call the loop.
commit_count = synthesize_loop(x, multiple_prs, change_pusher,
synthesizer)
if commit_count == 0:
logger.info("No changes. :)")
sys.exit(EXIT_CODE_SKIPPED)
return commit_count
finally:
if args.synth_path:
# We're generating code in a mono repo. The state left behind will
# probably be useful for generating the next API.
pass
else:
# We're generating a single API in a single repo, and using a different
# repo to generate the next API. So the next synth will not be able to
# use any of this state. Clean it up to avoid running out of disk space.
executor.run(["git", "clean", "-fdx"], cwd=working_repo_path)
def select_shard(config, shard: str):
shard_number, shard_count = map(int, shard.split("/"))
logger.info(f"Selecting shard {shard_number} of {shard_count}.")
config.sort(key=lambda repo: repo["name"])
return shard_list(config, shard_count)[shard_number]