def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config")
parser.add_argument("--github-token",
default=os.environ.get("GITHUB_TOKEN"))
parser.add_argument("extra_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
config = load_config(args.config)
if config is None:
sys.exit("No configuration could be loaded.")
gh = github.GitHub(args.github_token)
base_log_path = find_base_log_path()
results = synthesize_libraries(config, gh, args.github_token,
args.extra_args[1:], base_log_path)
num_failures = len([result for result in results if result["error"]])
if num_failures > 0:
logger.error(f"Failed to synthesize {num_failures} job(s).")
failure_percent = 100 * num_failures / len(results)
if failure_percent < 10:
pass # It's most likely an issue with a few APIs.
else:
sys.exit(1) # Raise the attention of autosynth maintainers.
def synthesize_library(
library: typing.Dict,
github_token: str,
extra_args: typing.List[str],
base_log_path: pathlib.Path,
runner: Runner = _execute,
) -> typing.Dict:
"""Run autosynth on a single library.
Arguments:
library {dict} - Library configuration
"""
logger.info(f"Synthesizing {library['name']}.")
command = [sys.executable, "-m", "autosynth.synth"]
env = os.environ
env["GITHUB_TOKEN"] = github_token
library_args = [
"--repository",
library["repository"],
"--synth-path",
library.get("synth-path", ""),
"--branch-suffix",
library.get("branch-suffix", ""),
"--pr-title",
library.get("pr-title", ""),
]
if library.get("metadata-path"):
library_args.extend(["--metadata-path", library.get("metadata-path")])
if library.get("deprecated-execution", False):
library_args.append("--deprecated-execution")
log_file_path = base_log_path / library["repository"] / "sponge_log.log"
# run autosynth in a separate process
(returncode, output) = runner(
command + library_args + library.get("args", []) + extra_args,
env,
log_file_path,
)
error = returncode not in (0, synth.EXIT_CODE_SKIPPED)
skipped = returncode == synth.EXIT_CODE_SKIPPED
if error:
logger.error(f"Synthesis failed for {library['name']}")
return {
"name": library["name"],
"output": output,
"error": error,
"skipped": skipped,
}
def synthesize(self,
log_file_path: pathlib.Path,
environ: typing.Mapping[str, str] = None) -> str:
"""
Returns:
The log of the call to synthtool.
"""
logger.info("Running synthtool")
if not self.deprecated_execution:
command = [
sys.executable,
"-m",
"synthtool",
"--metadata",
self.metadata_path,
self.synth_py_path,
"--",
]
else:
# Execute the synthesis script directly (deprecated)
command = [sys.executable, self.synth_py_path]
logger.info(command)
logger.debug(f"log_file_path: {log_file_path}")
# Ensure the logfile directory exists
log_file_path.parent.mkdir(parents=True, exist_ok=True)
# Tee the output into a provided location so we can see the return the final output
tee_proc = subprocess.Popen(["tee", log_file_path],
stdin=subprocess.PIPE)
# Invoke synth.py.
synth_proc = executor.run(
command + self.extra_args,
stderr=subprocess.STDOUT,
stdout=tee_proc.stdin,
env=(environ or os.environ),
universal_newlines=True,
)
if synth_proc.returncode:
logger.error("Synthesis failed")
synth_proc.check_returncode() # Raise an exception.
with open(log_file_path, "rt") as fp:
return fp.read()
def synthesize_loop(
toolbox: SynthesizeLoopToolbox,
multiple_prs: bool,
change_pusher: AbstractChangePusher,
synthesizer: AbstractSynthesizer,
) -> int:
"""Loops through all source versions and creates a commit for every version
changed that caused a change in the generated code.
Arguments:
toolbox {SynthesizeLoopToolbox} -- a toolbox
multiple_prs {bool} -- True to create one pull request per source.
change_pusher {AbstractChangePusher} -- Used to push changes to github.
synthesizer {AbstractSynthesizer} -- Invokes synthesize.
Returns:
int -- Number of commits committed to this repo.
"""
if not toolbox.versions:
return 0 # No versions, nothing to synthesize.
try:
if multiple_prs:
commit_count = 0
for fork in toolbox.fork():
if change_pusher.check_if_pr_already_exists(fork.branch):
continue
executor.check_call(["git", "checkout", fork.branch])
synthesize_inner_loop(fork, synthesizer)
commit_count += fork.commit_count
if fork.source_name == "self" or fork.count_commits_with_context(
) > 0:
fork.push_changes(change_pusher)
return commit_count
except Exception as e:
logger.error(e)
pass # Fall back to non-forked loop below.
if change_pusher.check_if_pr_already_exists(toolbox.branch):
return 0
synthesize_inner_loop(toolbox, synthesizer)
toolbox.push_changes(change_pusher)
return toolbox.commit_count
def _get_json_or_raise_exception(
response: requests.Response) -> Union[Dict, List]:
"""Helper to parse json response from GitHub.
If the response error code indicates an error (400+), try to log the
error message if pressent.
Arguments:
response {requests.Response} - The HTTP response object
Returns:
Union[Dict, List] - Parsed json data
"""
try:
json = response.json()
if response.status_code >= 400:
message = json.get("message")
if message:
logger.error(
f"Error making request ({response.status_code}): {message}"
)
else:
logger.error(f"Error making request ({response.status_code})")
logger.debug(json)
response.raise_for_status()
return json
except ValueError as e:
logger.error(f"Error parsing response json: {e}")
raise
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config")
parser.add_argument("--github-token",
default=os.environ.get("GITHUB_TOKEN"))
parser.add_argument(
"--shard",
default=os.environ.get("MULTISYNTH_SHARD"),
help=
"From the list of repos, the shard number and shard count, separated by a forward slash. Examples:\n0/10\n1/2",
)
parser.add_argument("extra_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
config = load_config(args.config)
if config is None:
sys.exit("No configuration could be loaded.")
if args.shard:
config = select_shard(config, args.shard)
gh = github.GitHub(args.github_token)
base_log_path = find_base_log_path()
results = synthesize_libraries(config, gh, args.github_token,
args.extra_args[1:], base_log_path)
num_failures = len([result for result in results if result["error"]])
if num_failures > 0:
logger.error(f"Failed to synthesize {num_failures} job(s).")
failure_percent = 100 * num_failures / len(results)
if failure_percent < 12:
pass # It's most likely an issue with a few APIs.
else:
sys.exit(1) # Raise the attention of autosynth maintainers.
def synthesize_library(
library: typing.Dict,
github_token: str,
extra_args: typing.List[str],
base_log_path: pathlib.Path,
runner: Runner = _execute,
) -> typing.Dict:
"""Run autosynth on a single library.
Arguments:
library {dict} - Library configuration
"""
logger.info(f"Synthesizing {library['name']}.")
command = [sys.executable, "-m", "autosynth.synth"]
env = os.environ
env["GITHUB_TOKEN"] = github_token
library_args = [
"--repository",
library["repository"],
"--synth-path",
library.get("synth-path", ""),
"--branch-suffix",
library.get("branch-suffix", ""),
"--pr-title",
library.get("pr-title", ""),
"--base-log-dir",
str(base_log_path),
]
if library.get("metadata-path"):
library_args.extend(["--metadata-path", library.get("metadata-path")])
if library.get("deprecated-execution", False):
library_args.append("--deprecated-execution")
log_file_dir = (pathlib.Path(base_log_path) / pathlib.Path(
library.get("synth-path", "") or library["repository"]).name)
log_file_path = log_file_dir / "sponge_log.log"
# run autosynth in a separate process
(returncode, output) = runner(
command + library_args + library.get("args", []) + extra_args,
env,
log_file_path,
)
error = returncode not in (0, synth.EXIT_CODE_SKIPPED)
skipped = returncode == synth.EXIT_CODE_SKIPPED
# Leave a sponge_log.xml side-by-side with sponge_log.log, and sponge
# will understand they're for the same task and render them accordingly.
results = [{
"name": library["name"],
"error": error,
"output": "See the test log.",
"skipped": skipped,
}]
make_report(library["name"], results, log_file_dir)
if error:
logger.error(f"Synthesis failed for {library['name']}")
return {
"name": library["name"],
"output": output.decode("utf-8"),
"error": error,
"skipped": skipped,
}
def synthesize_loop(
toolbox: SynthesizeLoopToolbox,
multiple_prs: bool,
change_pusher: AbstractChangePusher,
synthesizer: AbstractSynthesizer,
) -> int:
"""Loops through all source versions and creates a commit for every version
changed that caused a change in the generated code.
Arguments:
toolbox {SynthesizeLoopToolbox} -- a toolbox
multiple_prs {bool} -- True to create one pull request per source.
change_pusher {AbstractChangePusher} -- Used to push changes to github.
synthesizer {AbstractSynthesizer} -- Invokes synthesize.
Returns:
int -- Number of commits committed to this repo.
"""
if not toolbox.versions:
return 0 # No versions, nothing to synthesize.
# Synthesize the library with the most recent versions of all sources.
youngest = len(toolbox.versions) - 1
has_changes = toolbox.synthesize_version_in_new_branch(
synthesizer, youngest)
if not has_changes:
if (not toolbox.metadata_contains_generated_files(toolbox.branch)
and toolbox.metadata_contains_generated_files(
toolbox.sub_branch(youngest)) and
not change_pusher.check_if_pr_already_exists(toolbox.branch)):
# Special case: the repo owner turned on obsolete file tracking.
# Generate a one-time PR containing only metadata changes.
executor.check_call(["git", "checkout", toolbox.branch])
executor.check_call(
["git", "merge", "--squash",
toolbox.sub_branch(youngest)])
pr_title = "chore: start tracking obsolete files"
executor.check_call(["git", "commit", "-m", pr_title])
pr = change_pusher.push_changes(1, toolbox.branch, pr_title)
pr.add_labels(["context: full"])
return 1
return 0 # No changes, nothing to do.
try:
if multiple_prs:
commit_count = 0
for fork in toolbox.fork():
if change_pusher.check_if_pr_already_exists(fork.branch):
continue
executor.check_call(["git", "checkout", fork.branch])
synthesize_inner_loop(fork, synthesizer)
commit_count += fork.commit_count
if fork.source_name == "self" or fork.count_commits_with_context(
) > 0:
fork.push_changes(change_pusher)
return commit_count
except Exception as e:
logger.error(e)
# Fallback to the single_pr loop to try to make some progress.
synthesize_loop_single_pr(toolbox, change_pusher, synthesizer)
# But still report the failure.
raise
return synthesize_loop_single_pr(toolbox, change_pusher, synthesizer)
