def main(): parser = argparse.ArgumentParser() parser.add_argument("--config") parser.add_argument("--github-token", default=os.environ.get("GITHUB_TOKEN")) parser.add_argument("extra_args", nargs=argparse.REMAINDER) args = parser.parse_args() config = load_config(args.config) if config is None: sys.exit("No configuration could be loaded.") gh = github.GitHub(args.github_token) base_log_path = find_base_log_path() results = synthesize_libraries(config, gh, args.github_token, args.extra_args[1:], base_log_path) num_failures = len([result for result in results if result["error"]]) if num_failures > 0: logger.error(f"Failed to synthesize {num_failures} job(s).") failure_percent = 100 * num_failures / len(results) if failure_percent < 10: pass # It's most likely an issue with a few APIs. else: sys.exit(1) # Raise the attention of autosynth maintainers.
def synthesize_library( library: typing.Dict, github_token: str, extra_args: typing.List[str], base_log_path: pathlib.Path, runner: Runner = _execute, ) -> typing.Dict: """Run autosynth on a single library. Arguments: library {dict} - Library configuration """ logger.info(f"Synthesizing {library['name']}.") command = [sys.executable, "-m", "autosynth.synth"] env = os.environ env["GITHUB_TOKEN"] = github_token library_args = [ "--repository", library["repository"], "--synth-path", library.get("synth-path", ""), "--branch-suffix", library.get("branch-suffix", ""), "--pr-title", library.get("pr-title", ""), ] if library.get("metadata-path"): library_args.extend(["--metadata-path", library.get("metadata-path")]) if library.get("deprecated-execution", False): library_args.append("--deprecated-execution") log_file_path = base_log_path / library["repository"] / "sponge_log.log" # run autosynth in a separate process (returncode, output) = runner( command + library_args + library.get("args", []) + extra_args, env, log_file_path, ) error = returncode not in (0, synth.EXIT_CODE_SKIPPED) skipped = returncode == synth.EXIT_CODE_SKIPPED if error: logger.error(f"Synthesis failed for {library['name']}") return { "name": library["name"], "output": output, "error": error, "skipped": skipped, }
def synthesize(self, log_file_path: pathlib.Path, environ: typing.Mapping[str, str] = None) -> str: """ Returns: The log of the call to synthtool. """ logger.info("Running synthtool") if not self.deprecated_execution: command = [ sys.executable, "-m", "synthtool", "--metadata", self.metadata_path, self.synth_py_path, "--", ] else: # Execute the synthesis script directly (deprecated) command = [sys.executable, self.synth_py_path] logger.info(command) logger.debug(f"log_file_path: {log_file_path}") # Ensure the logfile directory exists log_file_path.parent.mkdir(parents=True, exist_ok=True) # Tee the output into a provided location so we can see the return the final output tee_proc = subprocess.Popen(["tee", log_file_path], stdin=subprocess.PIPE) # Invoke synth.py. synth_proc = executor.run( command + self.extra_args, stderr=subprocess.STDOUT, stdout=tee_proc.stdin, env=(environ or os.environ), universal_newlines=True, ) if synth_proc.returncode: logger.error("Synthesis failed") synth_proc.check_returncode() # Raise an exception. with open(log_file_path, "rt") as fp: return fp.read()
def synthesize_loop( toolbox: SynthesizeLoopToolbox, multiple_prs: bool, change_pusher: AbstractChangePusher, synthesizer: AbstractSynthesizer, ) -> int: """Loops through all source versions and creates a commit for every version changed that caused a change in the generated code. Arguments: toolbox {SynthesizeLoopToolbox} -- a toolbox multiple_prs {bool} -- True to create one pull request per source. change_pusher {AbstractChangePusher} -- Used to push changes to github. synthesizer {AbstractSynthesizer} -- Invokes synthesize. Returns: int -- Number of commits committed to this repo. """ if not toolbox.versions: return 0 # No versions, nothing to synthesize. try: if multiple_prs: commit_count = 0 for fork in toolbox.fork(): if change_pusher.check_if_pr_already_exists(fork.branch): continue executor.check_call(["git", "checkout", fork.branch]) synthesize_inner_loop(fork, synthesizer) commit_count += fork.commit_count if fork.source_name == "self" or fork.count_commits_with_context( ) > 0: fork.push_changes(change_pusher) return commit_count except Exception as e: logger.error(e) pass # Fall back to non-forked loop below. if change_pusher.check_if_pr_already_exists(toolbox.branch): return 0 synthesize_inner_loop(toolbox, synthesizer) toolbox.push_changes(change_pusher) return toolbox.commit_count
def _get_json_or_raise_exception( response: requests.Response) -> Union[Dict, List]: """Helper to parse json response from GitHub. If the response error code indicates an error (400+), try to log the error message if pressent. Arguments: response {requests.Response} - The HTTP response object Returns: Union[Dict, List] - Parsed json data """ try: json = response.json() if response.status_code >= 400: message = json.get("message") if message: logger.error( f"Error making request ({response.status_code}): {message}" ) else: logger.error(f"Error making request ({response.status_code})") logger.debug(json) response.raise_for_status() return json except ValueError as e: logger.error(f"Error parsing response json: {e}") raise
def main(): parser = argparse.ArgumentParser() parser.add_argument("--config") parser.add_argument("--github-token", default=os.environ.get("GITHUB_TOKEN")) parser.add_argument( "--shard", default=os.environ.get("MULTISYNTH_SHARD"), help= "From the list of repos, the shard number and shard count, separated by a forward slash. Examples:\n0/10\n1/2", ) parser.add_argument("extra_args", nargs=argparse.REMAINDER) args = parser.parse_args() config = load_config(args.config) if config is None: sys.exit("No configuration could be loaded.") if args.shard: config = select_shard(config, args.shard) gh = github.GitHub(args.github_token) base_log_path = find_base_log_path() results = synthesize_libraries(config, gh, args.github_token, args.extra_args[1:], base_log_path) num_failures = len([result for result in results if result["error"]]) if num_failures > 0: logger.error(f"Failed to synthesize {num_failures} job(s).") failure_percent = 100 * num_failures / len(results) if failure_percent < 12: pass # It's most likely an issue with a few APIs. else: sys.exit(1) # Raise the attention of autosynth maintainers.
def synthesize_library( library: typing.Dict, github_token: str, extra_args: typing.List[str], base_log_path: pathlib.Path, runner: Runner = _execute, ) -> typing.Dict: """Run autosynth on a single library. Arguments: library {dict} - Library configuration """ logger.info(f"Synthesizing {library['name']}.") command = [sys.executable, "-m", "autosynth.synth"] env = os.environ env["GITHUB_TOKEN"] = github_token library_args = [ "--repository", library["repository"], "--synth-path", library.get("synth-path", ""), "--branch-suffix", library.get("branch-suffix", ""), "--pr-title", library.get("pr-title", ""), "--base-log-dir", str(base_log_path), ] if library.get("metadata-path"): library_args.extend(["--metadata-path", library.get("metadata-path")]) if library.get("deprecated-execution", False): library_args.append("--deprecated-execution") log_file_dir = (pathlib.Path(base_log_path) / pathlib.Path( library.get("synth-path", "") or library["repository"]).name) log_file_path = log_file_dir / "sponge_log.log" # run autosynth in a separate process (returncode, output) = runner( command + library_args + library.get("args", []) + extra_args, env, log_file_path, ) error = returncode not in (0, synth.EXIT_CODE_SKIPPED) skipped = returncode == synth.EXIT_CODE_SKIPPED # Leave a sponge_log.xml side-by-side with sponge_log.log, and sponge # will understand they're for the same task and render them accordingly. results = [{ "name": library["name"], "error": error, "output": "See the test log.", "skipped": skipped, }] make_report(library["name"], results, log_file_dir) if error: logger.error(f"Synthesis failed for {library['name']}") return { "name": library["name"], "output": output.decode("utf-8"), "error": error, "skipped": skipped, }
def synthesize_loop( toolbox: SynthesizeLoopToolbox, multiple_prs: bool, change_pusher: AbstractChangePusher, synthesizer: AbstractSynthesizer, ) -> int: """Loops through all source versions and creates a commit for every version changed that caused a change in the generated code. Arguments: toolbox {SynthesizeLoopToolbox} -- a toolbox multiple_prs {bool} -- True to create one pull request per source. change_pusher {AbstractChangePusher} -- Used to push changes to github. synthesizer {AbstractSynthesizer} -- Invokes synthesize. Returns: int -- Number of commits committed to this repo. """ if not toolbox.versions: return 0 # No versions, nothing to synthesize. # Synthesize the library with the most recent versions of all sources. youngest = len(toolbox.versions) - 1 has_changes = toolbox.synthesize_version_in_new_branch( synthesizer, youngest) if not has_changes: if (not toolbox.metadata_contains_generated_files(toolbox.branch) and toolbox.metadata_contains_generated_files( toolbox.sub_branch(youngest)) and not change_pusher.check_if_pr_already_exists(toolbox.branch)): # Special case: the repo owner turned on obsolete file tracking. # Generate a one-time PR containing only metadata changes. executor.check_call(["git", "checkout", toolbox.branch]) executor.check_call( ["git", "merge", "--squash", toolbox.sub_branch(youngest)]) pr_title = "chore: start tracking obsolete files" executor.check_call(["git", "commit", "-m", pr_title]) pr = change_pusher.push_changes(1, toolbox.branch, pr_title) pr.add_labels(["context: full"]) return 1 return 0 # No changes, nothing to do. try: if multiple_prs: commit_count = 0 for fork in toolbox.fork(): if change_pusher.check_if_pr_already_exists(fork.branch): continue executor.check_call(["git", "checkout", fork.branch]) synthesize_inner_loop(fork, synthesizer) commit_count += fork.commit_count if fork.source_name == "self" or fork.count_commits_with_context( ) > 0: fork.push_changes(change_pusher) return commit_count except Exception as e: logger.error(e) # Fallback to the single_pr loop to try to make some progress. synthesize_loop_single_pr(toolbox, change_pusher, synthesizer) # But still report the failure. raise return synthesize_loop_single_pr(toolbox, change_pusher, synthesizer)