def list_apis() -> Dict[str, List[str]]: """List all the available discovery versions grouped by api""" gh = github.GitHub(os.environ["GITHUB_TOKEN"]) # List discovery documents from discovery-artifact-manager repo = "googleapis/discovery-artifact-manager" # Find the unique API names apis = {} files = gh.list_files(repo, "discoveries") discoveries = [ file["name"] for file in files if file["name"].endswith(".json") ] for discovery in discoveries: m = VERSION_REGEX.match(discovery) if m: api = m.group(1) version = m.group(2) if api not in apis: apis[api] = [] apis[api].append(version) return apis
def main(): parser = argparse.ArgumentParser() parser.add_argument("--config") parser.add_argument("--github-token", default=os.environ.get("GITHUB_TOKEN")) parser.add_argument("extra_args", nargs=argparse.REMAINDER) args = parser.parse_args() config = load_config(args.config) if config is None: sys.exit("No configuration could be loaded.") gh = github.GitHub(args.github_token) base_log_path = find_base_log_path() results = synthesize_libraries(config, gh, args.github_token, args.extra_args[1:], base_log_path) num_failures = len([result for result in results if result["error"]]) if num_failures > 0: logger.error(f"Failed to synthesize {num_failures} job(s).") failure_percent = 100 * num_failures / len(results) if failure_percent < 10: pass # It's most likely an issue with a few APIs. else: sys.exit(1) # Raise the attention of autosynth maintainers.
def list_repositories(): gh = github.GitHub(os.environ["GITHUB_TOKEN"]) repos = _get_repo_list_from_sloth(gh) repos = [repo for repo in repos if _is_nodejs_synth_repo(gh, repo)] return [{ "name": repo["repo"].split("/")[-1], "repository": repo["repo"] } for repo in repos]
def list_repositories(): """Finds repositories with a `synth.py` in the top-level""" gh = github.GitHub(os.environ["GITHUB_TOKEN"]) repos = _get_repo_list_from_sloth(gh) repos = [repo for repo in repos if _is_python_synth_repo(gh, repo)] repo_list = [{ "name": repo["repo"].split("/")[-1], "repository": repo["repo"] } for repo in repos] return repo_list
def list_split_repositories( repo_name_chunk: str, majority_languages: Sequence[str] = ()) -> List[Dict]: """List github repos for a programming language. Args: repo_name_chunk (str): return repos that have this chunk in the repo name. Example: "nodejs" majority_languages (Sequence[str], optional): return repos that have a majority of their code written in one of these programming languages. Example: ("JavaScript", "TypeScript") Returns: List[Dict]: [description] """ gh = github.GitHub(os.environ["GITHUB_TOKEN"]) all_repos = set([ repo["name"] for repo in gh.list_repos("googleapis") if not repo["archived"] ]) # Find repos with the language as part of the repo name. lang_repos = set( [repo for repo in all_repos if repo_name_chunk in repo.split("-")]) if majority_languages: # Ignore all repos whose name tags them for a language. silver_name_chunks = set(_SILVER_NAME_CHUNKS) all_lang_repos = set([ repo for repo in all_repos if silver_name_chunks.intersection(set(repo.split("-"))) ]) # Find repos with the majority of their code written in the language. silver_language_names = set(_SILVER_LANGUAGE_NAMES) for repo in all_repos - all_lang_repos: languages = gh.get_languages(f"googleapis/{repo}") ranks = [ (count, lang) for (lang, count) in languages.items() # Ignore languages we don't care about, like Shell. if lang in silver_language_names ] if ranks and max(ranks)[1] in majority_languages: lang_repos.add(repo) synth_repos = [] for repo in sorted(lang_repos): # Only repos with a synth.py in the top-level directory. if not gh.list_files(f"googleapis/{repo}", "synth.py"): continue synth_repos.append({"name": repo, "repository": f"googleapis/{repo}"}) return synth_repos
def list_repositories(): """List all the available discovery versions grouped by api""" gh = github.GitHub(os.environ["GITHUB_TOKEN"]) repo = "googleapis/elixir-google-api" api_config = gh.get_contents(repository=repo, path="config/apis.json") api_names = set() for api in json.loads(api_config): api_names.add(api["name"]) return [{ "name": api, "repository": "googleapis/elixir-google-api", "branch-suffix": api.lower(), "pr-title": f"Regenerate {api} client", "args": [api], "metadata-path": f"clients/{_underscore(api)}", } for api in sorted(api_names)]
def list_repositories(): gh = github.GitHub(os.environ["GITHUB_TOKEN"]) # Presently this only enumerates folders from the google-cloud-dotnet # monorepo, under the autosynth directory. repo = "googleapis/google-cloud-dotnet" clients = gh.list_files(repo, "apis") subdirs = [item["path"] for item in clients if item["type"] == "dir"] # No hidden dirs subdirs = [subdir for subdir in subdirs if not subdir.startswith(".")] # Only subdirs that have synth.py files. subdirs = [ subdir for subdir in subdirs if gh.check_for_file(repo, f"{subdir}/synth.py") ] return [_config_for_subdir(repo, subdir) for subdir in subdirs]
def list_repositories(): gh = github.GitHub(os.environ["GITHUB_TOKEN"]) # Presently this only enumerates folders from the google-cloud-php # monorepo. repo = "googleapis/google-cloud-php" tree = gh.get_tree(repo) subdirs = [item["path"] for item in tree["tree"] if item["type"] == "tree"] # No hidden dirs subdirs = [subdir for subdir in subdirs if not subdir.startswith(".")] # Only subdirs that have synth.py files. subdirs = [ subdir for subdir in subdirs if gh.check_for_file(repo, f"{subdir}/synth.py") ] return [{ "name": subdir.lower(), "repository": repo, "synth-path": subdir, "branch-suffix": subdir.lower(), } for subdir in subdirs]
def main(): parser = argparse.ArgumentParser() parser.add_argument("--config") parser.add_argument("--github-token", default=os.environ.get("GITHUB_TOKEN")) parser.add_argument( "--shard", default=os.environ.get("MULTISYNTH_SHARD"), help= "From the list of repos, the shard number and shard count, separated by a forward slash. Examples:\n0/10\n1/2", ) parser.add_argument("extra_args", nargs=argparse.REMAINDER) args = parser.parse_args() config = load_config(args.config) if config is None: sys.exit("No configuration could be loaded.") if args.shard: config = select_shard(config, args.shard) gh = github.GitHub(args.github_token) base_log_path = find_base_log_path() results = synthesize_libraries(config, gh, args.github_token, args.extra_args[1:], base_log_path) num_failures = len([result for result in results if result["error"]]) if num_failures > 0: logger.error(f"Failed to synthesize {num_failures} job(s).") failure_percent = 100 * num_failures / len(results) if failure_percent < 12: pass # It's most likely an issue with a few APIs. else: sys.exit(1) # Raise the attention of autosynth maintainers.
def list_repositories(): gh = github.GitHub(os.environ["GITHUB_TOKEN"]) # Presently this only enumerates folders from the google-cloud-ruby # monorepo. repo = "googleapis/google-cloud-ruby" default_branch = gh.get_default_branch(repo) tree = gh.get_tree(repo, default_branch) subdirs = [item["path"] for item in tree["tree"] if item["type"] == "tree"] # No hidden dirs subdirs = [subdir for subdir in subdirs if not subdir.startswith(".")] # Only subdirs that have synth.py files. subdirs = [subdir for subdir in subdirs if _check_bazel_synth(gh, repo, subdir)] return [ { "name": subdir.replace("google-cloud-", ""), "repository": repo, "synth-path": subdir, "branch-suffix": subdir.replace("google-cloud-", ""), } for subdir in subdirs ]
def _inner_main(temp_dir: str) -> int: """ Returns: int -- Number of commits committed to the repo. """ parser = argparse.ArgumentParser() parser.add_argument("--github-user", default=os.environ.get("GITHUB_USER")) parser.add_argument("--github-email", default=os.environ.get("GITHUB_EMAIL")) parser.add_argument("--github-token", default=os.environ.get("GITHUB_TOKEN")) parser.add_argument("--repository", default=os.environ.get("REPOSITORY"), required=True) parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH")) parser.add_argument("--metadata-path", default=os.environ.get("METADATA_PATH")) parser.add_argument( "--deprecated-execution", default=False, action="store_true", help= "If specified, execute synth.py directly instead of synthtool. This behavior is deprecated.", ) parser.add_argument("--branch-suffix", default=os.environ.get("BRANCH_SUFFIX", None)) parser.add_argument("--pr-title", default="") parser.add_argument("extra_args", nargs=argparse.REMAINDER) args = parser.parse_args() gh = github.GitHub(args.github_token) branch = "-".join(filter(None, ["autosynth", args.branch_suffix])) pr_title = args.pr_title or ( f"[CHANGE ME] Re-generated {args.synth_path or ''} to pick up changes in " f"the API or client library generator.") change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh, branch) # capture logs for later base_synth_log_path = pathlib.Path( os.path.realpath("./logs")) / args.repository if args.synth_path: base_synth_log_path /= args.synth_path logger.info(f"logs will be written to: {base_synth_log_path}") working_repo_path = synthtool_git.clone( f"https://github.com/{args.repository}.git") try: os.chdir(working_repo_path) git.configure_git(args.github_user, args.github_email) git.setup_branch(branch) if args.synth_path: os.chdir(args.synth_path) metadata_path = os.path.join(args.metadata_path or "", "synth.metadata") flags = autosynth.flags.parse_flags() # Override flags specified in synth.py with flags specified in environment vars. for key in flags.keys(): env_value = os.environ.get(key, "") if env_value: flags[key] = False if env_value.lower( ) == "false" else env_value metadata = load_metadata(metadata_path) multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS] multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS] if (not multiple_commits and not multiple_prs) or not metadata: if change_pusher.check_if_pr_already_exists(branch): return 0 synth_log = Synthesizer( metadata_path, args.extra_args, deprecated_execution=args.deprecated_execution, ).synthesize(base_synth_log_path) if not has_changes(): logger.info("No changes. :)") sys.exit(EXIT_CODE_SKIPPED) git.commit_all_changes(pr_title) change_pusher.push_changes(1, branch, pr_title, synth_log) return 1 else: if not multiple_prs and change_pusher.check_if_pr_already_exists( branch): return 0 # There's already an existing PR # Enumerate the versions to loop over. sources = metadata.get("sources", []) source_versions = [ git_source.enumerate_versions_for_working_repo( metadata_path, sources) ] # Add supported source version types below: source_versions.extend( git_source.enumerate_versions(sources, pathlib.Path(temp_dir))) # Prepare to call synthesize loop. synthesizer = Synthesizer( metadata_path, args.extra_args, args.deprecated_execution, "synth.py", ) x = SynthesizeLoopToolbox( source_versions, branch, temp_dir, metadata_path, args.synth_path, base_synth_log_path, ) if not multiple_commits: change_pusher = SquashingChangePusher(change_pusher) # Call the loop. commit_count = synthesize_loop(x, multiple_prs, change_pusher, synthesizer) if commit_count == 0: logger.info("No changes. :)") sys.exit(EXIT_CODE_SKIPPED) return commit_count finally: if args.synth_path: # We're generating code in a mono repo. The state left behind will # probably be useful for generating the next API. pass else: # We're generating a single API in a single repo, and using a different # repo to generate the next API. So the next synth will not be able to # use any of this state. Clean it up to avoid running out of disk space. executor.run(["git", "clean", "-fdx"], cwd=working_repo_path)
def new_gh(): return github.GitHub(os.environ["GITHUB_TOKEN"])
def list_repositories(): gh = github.GitHub(os.environ["GITHUB_TOKEN"]) return _google_cloud_java_repos(gh) + _other_repos(gh)