def git_branches_differ(branch_a: str, branch_b: str, metadata_path: str) -> bool: # Check to see if any files besides synth.metadata were added, modified, deleted. diff_cmd = ["git", "diff", f"{branch_a}..{branch_b}"] diff_cmd.extend(["--", ".", f":(exclude){metadata_path}"]) proc = executor.run(diff_cmd, stdout=subprocess.PIPE, universal_newlines=True) proc.check_returncode() if bool(proc.stdout): return True # Check to see if synth.metadata was added. proc = executor.run( ["git", "diff", f"{branch_a}..{branch_b}", "--", metadata_path], stdout=subprocess.PIPE, universal_newlines=True, ) proc.check_returncode() diff_text = proc.stdout pattern = "^--- /dev/null" return bool(re.search(pattern, diff_text, re.MULTILINE))
def get_last_commit_to_file(file_path: str) -> str: """Returns the commit hash of the most recent change to a file.""" parent_dir = pathlib.Path(file_path).parent proc = executor.run( ["git", "log", "--pretty=format:%H", "-1", "--no-decorate", file_path], stdout=subprocess.PIPE, universal_newlines=True, cwd=parent_dir, ) proc.check_returncode() return proc.stdout.strip()
def get_timestamp(self) -> datetime.datetime: if self.timestamp is None: unix_timestamp = executor.run( ["git", "log", "-1", "--pretty=%at", self.sha], cwd=self.repo_path, universal_newlines=True, check=True, stdout=subprocess.PIPE, ).stdout.strip() self.timestamp = datetime.datetime.fromtimestamp(float(unix_timestamp)) return self.timestamp
def get_comment(self) -> str: # Construct a comment using the text of the git commit. if self.comment is None: pretty = "--pretty=%B%n%nSource-Author: %an <%ae>%nSource-Date: %ad" git_log: str = executor.run( ["git", "log", self.sha, "-1", "--no-decorate", pretty], cwd=self.repo_path, stdout=subprocess.PIPE, universal_newlines=True, check=True, ).stdout.strip() self.comment = _compose_comment(self.remote, self.sha, git_log) return self.comment
def enumerate_versions_for_working_repo( metadata_path: str, sources: typing.List[typing.Dict[str, typing.Dict]] ) -> typing.List[autosynth.abstract_source.AbstractSourceVersion]: """Enumerates every commit after the most recent commit to metadata_path. Special case for enumerating the change history of the repo that we're actually generating. Arguments: metadata_path {str} -- Metadata file path. Returns: typing.List[autosynth.abstract_source.AbstractSourceVersion] -- versions """ # Get the repo root directory that contains metadata_path. local_repo_dir = git.get_repo_root_dir(metadata_path) # Find the most recent commit hash. head_sha = executor.run( ["git", "log", "-1", "--pretty=%H"], stdout=subprocess.PIPE, universal_newlines=True, cwd=local_repo_dir, check=True, ).stdout.strip() # Get the remote url. remote = executor.run( ["git", "remote", "get-url", "origin"], stdout=subprocess.PIPE, universal_newlines=True, cwd=local_repo_dir, check=True, ).stdout.strip() desc = f"This git repo ({remote})" version = GitSourceVersion(local_repo_dir, head_sha, remote, desc, "self") # The change from this repository must always be built first. version.timestamp = datetime.datetime.fromtimestamp(0) return [version]
def _execute(command: typing.List[str], env: typing.Any, log_file_path: pathlib.Path) -> typing.Tuple[int, bytes]: """Helper to wrap command invocation for testing""" # Ensure the logfile directory exists log_file_path.parent.mkdir(parents=True, exist_ok=True) with open(log_file_path, "wb+") as log_file: result = executor.run( command=command, stdout=log_file, stderr=subprocess.STDOUT, check=False, encoding="utf-8", env=env, ) with open(log_file_path, "rb") as fp: return (result.returncode, fp.read())
def synthesize(self, log_file_path: pathlib.Path, environ: typing.Mapping[str, str] = None) -> str: """ Returns: The log of the call to synthtool. """ logger.info("Running synthtool") if not self.deprecated_execution: command = [ sys.executable, "-m", "synthtool", "--metadata", self.metadata_path, self.synth_py_path, "--", ] else: # Execute the synthesis script directly (deprecated) command = [sys.executable, self.synth_py_path] logger.info(command) logger.debug(f"log_file_path: {log_file_path}") # Ensure the logfile directory exists log_file_path.parent.mkdir(parents=True, exist_ok=True) # Tee the output into a provided location so we can see the return the final output tee_proc = subprocess.Popen(["tee", log_file_path], stdin=subprocess.PIPE) # Invoke synth.py. synth_proc = executor.run( command + self.extra_args, stderr=subprocess.STDOUT, stdout=tee_proc.stdin, env=(environ or os.environ), universal_newlines=True, ) if synth_proc.returncode: logger.error("Synthesis failed") synth_proc.check_returncode() # Raise an exception. with open(log_file_path, "rt") as fp: return fp.read()
def get_commit_subject(repo_dir: str = None, sha: str = None) -> str: """Gets the subject line of the a commit. Keyword Arguments: repo_dir {str} -- a directory in the repo; None means use cwd. (default: {None}) sha {str} -- the sha of the commit. None means the most recent commit. Returns: {str} -- the subject line """ commit_message: str = executor.run( ["git", "log", "-1", "--no-decorate", "--format=%B"] + ([sha] if sha else []), stdout=subprocess.PIPE, universal_newlines=True, check=True, cwd=repo_dir, ).stdout lines = commit_message.splitlines() return lines[0].strip() if lines else ""
def get_repo_root_dir(repo_path: str) -> str: """Given a path to a file or dir in a repo, find the root directory of the repo. Arguments: repo_path {str} -- Any path into the repo. Returns: str -- The repo's root directory. """ path = pathlib.Path(repo_path) if not path.is_dir(): path = path.parent proc = executor.run( ["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, universal_newlines=True, cwd=str(path), ) proc.check_returncode() return proc.stdout.strip()
def _collect_trailers(commit_count: int, git_dir: typing.Optional[str] = None) -> str: """Collects the trailers from recent commits in the repo. Only collects the two trailers we're interested in. Arguments: commit_count {int} -- Number of commits to collect trailers from. Keyword Arguments: git_dir {typing.Optional[str]} -- directory of git repo (default: {None}) Returns: str -- The trailer lines from the recent commits. """ text = executor.run( ["git", "log", f"-{commit_count}", "--pretty=%b"], universal_newlines=True, check=True, stdout=subprocess.PIPE, cwd=git_dir, ).stdout return _parse_trailers(text)
def get_commit_shas_since(sha: str, dir: str) -> typing.List[str]: """Gets the list of shas for commits committed after the given sha. Arguments: sha {str} -- The sha in the git history. dir {str} -- An absolute path to a directory in the git repository. Returns: typing.List[str] -- A list of shas. The 0th sha is sha argument (the oldest sha). """ proc = executor.run( ["git", "log", f"{sha}..HEAD", "--pretty=%H", "--no-decorate"], universal_newlines=True, stdout=subprocess.PIPE, cwd=dir, ) proc.check_returncode() shas = proc.stdout.split() shas.append(sha) shas.reverse() return shas
def _get_pretty(self, pretty: str) -> str: """Gets the pretty log for this commit. Arguments: pretty {str} -- the git log pretty format Returns: str -- the output of the git log command, stripped of leading and trailing whitespace """ git_log: str = executor.run( [ "git", "log", self.sha, "-1", "--no-decorate", f"--pretty={pretty}" ], cwd=self.repo_path, stdout=subprocess.PIPE, universal_newlines=True, check=True, ).stdout.strip() return git_log
def commit_all_changes(message: str) -> int: """Commits all changes in the repo. Args: message (str): the commit message Returns: int: 1 if a change was commited, otherwise 0. """ executor.check_call(["git", "add", "-A"]) status = executor.run( ["git", "status", "--porcelain"], universal_newlines=True, stdout=subprocess.PIPE, check=True, ).stdout.strip() if status: executor.check_call(["git", "commit", "-m", message]) return 1 else: # There are no changes to commit. return 0
def _inner_main(temp_dir: str) -> int: """ Returns: int -- Number of commits committed to the repo. """ parser = argparse.ArgumentParser() parser.add_argument("--github-user", default=os.environ.get("GITHUB_USER")) parser.add_argument("--github-email", default=os.environ.get("GITHUB_EMAIL")) parser.add_argument("--github-token", default=os.environ.get("GITHUB_TOKEN")) parser.add_argument("--repository", default=os.environ.get("REPOSITORY"), required=True) parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH")) parser.add_argument("--metadata-path", default=os.environ.get("METADATA_PATH")) parser.add_argument( "--deprecated-execution", default=False, action="store_true", help= "If specified, execute synth.py directly instead of synthtool. This behavior is deprecated.", ) parser.add_argument("--branch-suffix", default=os.environ.get("BRANCH_SUFFIX", None)) parser.add_argument("--pr-title", default="") parser.add_argument("extra_args", nargs=argparse.REMAINDER) args = parser.parse_args() gh = github.GitHub(args.github_token) branch = "-".join(filter(None, ["autosynth", args.branch_suffix])) pr_title = args.pr_title or ( f"[CHANGE ME] Re-generated {args.synth_path or ''} to pick up changes in " f"the API or client library generator.") change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh, branch) # capture logs for later base_synth_log_path = pathlib.Path( os.path.realpath("./logs")) / args.repository if args.synth_path: base_synth_log_path /= args.synth_path logger.info(f"logs will be written to: {base_synth_log_path}") working_repo_path = synthtool_git.clone( f"https://github.com/{args.repository}.git") try: os.chdir(working_repo_path) git.configure_git(args.github_user, args.github_email) git.setup_branch(branch) if args.synth_path: os.chdir(args.synth_path) metadata_path = os.path.join(args.metadata_path or "", "synth.metadata") flags = autosynth.flags.parse_flags() # Override flags specified in synth.py with flags specified in environment vars. for key in flags.keys(): env_value = os.environ.get(key, "") if env_value: flags[key] = False if env_value.lower( ) == "false" else env_value metadata = load_metadata(metadata_path) multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS] multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS] if (not multiple_commits and not multiple_prs) or not metadata: if change_pusher.check_if_pr_already_exists(branch): return 0 synth_log = Synthesizer( metadata_path, args.extra_args, deprecated_execution=args.deprecated_execution, ).synthesize(base_synth_log_path) if not has_changes(): logger.info("No changes. :)") sys.exit(EXIT_CODE_SKIPPED) git.commit_all_changes(pr_title) change_pusher.push_changes(1, branch, pr_title, synth_log) return 1 else: if not multiple_prs and change_pusher.check_if_pr_already_exists( branch): return 0 # There's already an existing PR # Enumerate the versions to loop over. sources = metadata.get("sources", []) source_versions = [ git_source.enumerate_versions_for_working_repo( metadata_path, sources) ] # Add supported source version types below: source_versions.extend( git_source.enumerate_versions(sources, pathlib.Path(temp_dir))) # Prepare to call synthesize loop. synthesizer = Synthesizer( metadata_path, args.extra_args, args.deprecated_execution, "synth.py", ) x = SynthesizeLoopToolbox( source_versions, branch, temp_dir, metadata_path, args.synth_path, base_synth_log_path, ) if not multiple_commits: change_pusher = SquashingChangePusher(change_pusher) # Call the loop. commit_count = synthesize_loop(x, multiple_prs, change_pusher, synthesizer) if commit_count == 0: logger.info("No changes. :)") sys.exit(EXIT_CODE_SKIPPED) return commit_count finally: if args.synth_path: # We're generating code in a mono repo. The state left behind will # probably be useful for generating the next API. pass else: # We're generating a single API in a single repo, and using a different # repo to generate the next API. So the next synth will not be able to # use any of this state. Clean it up to avoid running out of disk space. executor.run(["git", "clean", "-fdx"], cwd=working_repo_path)