def list_apis() -> Dict[str, List[str]]:
"""List all the available discovery versions grouped by api"""
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
# List discovery documents from discovery-artifact-manager
repo = "googleapis/discovery-artifact-manager"
# Find the unique API names
apis = {}
files = gh.list_files(repo, "discoveries")
discoveries = [
file["name"] for file in files if file["name"].endswith(".json")
]
for discovery in discoveries:
m = VERSION_REGEX.match(discovery)
if m:
api = m.group(1)
version = m.group(2)
if api not in apis:
apis[api] = []
apis[api].append(version)
return apis
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config")
parser.add_argument("--github-token",
default=os.environ.get("GITHUB_TOKEN"))
parser.add_argument("extra_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
config = load_config(args.config)
if config is None:
sys.exit("No configuration could be loaded.")
gh = github.GitHub(args.github_token)
base_log_path = find_base_log_path()
results = synthesize_libraries(config, gh, args.github_token,
args.extra_args[1:], base_log_path)
num_failures = len([result for result in results if result["error"]])
if num_failures > 0:
logger.error(f"Failed to synthesize {num_failures} job(s).")
failure_percent = 100 * num_failures / len(results)
if failure_percent < 10:
pass # It's most likely an issue with a few APIs.
else:
sys.exit(1) # Raise the attention of autosynth maintainers.
def list_repositories():
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
repos = _get_repo_list_from_sloth(gh)
repos = [repo for repo in repos if _is_nodejs_synth_repo(gh, repo)]
return [{
"name": repo["repo"].split("/")[-1],
"repository": repo["repo"]
} for repo in repos]
def list_repositories():
"""Finds repositories with a `synth.py` in the top-level"""
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
repos = _get_repo_list_from_sloth(gh)
repos = [repo for repo in repos if _is_python_synth_repo(gh, repo)]
repo_list = [{
"name": repo["repo"].split("/")[-1],
"repository": repo["repo"]
} for repo in repos]
return repo_list
def list_split_repositories(
repo_name_chunk: str,
majority_languages: Sequence[str] = ()) -> List[Dict]:
"""List github repos for a programming language.
Args:
repo_name_chunk (str): return repos that have this chunk in the repo name.
Example: "nodejs"
majority_languages (Sequence[str], optional): return repos that have a majority
of their code written in one of these programming languages.
Example: ("JavaScript", "TypeScript")
Returns:
List[Dict]: [description]
"""
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
all_repos = set([
repo["name"] for repo in gh.list_repos("googleapis")
if not repo["archived"]
])
# Find repos with the language as part of the repo name.
lang_repos = set(
[repo for repo in all_repos if repo_name_chunk in repo.split("-")])
if majority_languages:
# Ignore all repos whose name tags them for a language.
silver_name_chunks = set(_SILVER_NAME_CHUNKS)
all_lang_repos = set([
repo for repo in all_repos
if silver_name_chunks.intersection(set(repo.split("-")))
])
# Find repos with the majority of their code written in the language.
silver_language_names = set(_SILVER_LANGUAGE_NAMES)
for repo in all_repos - all_lang_repos:
languages = gh.get_languages(f"googleapis/{repo}")
ranks = [
(count, lang) for (lang, count) in languages.items()
# Ignore languages we don't care about, like Shell.
if lang in silver_language_names
]
if ranks and max(ranks)[1] in majority_languages:
lang_repos.add(repo)
synth_repos = []
for repo in sorted(lang_repos):
# Only repos with a synth.py in the top-level directory.
if not gh.list_files(f"googleapis/{repo}", "synth.py"):
continue
synth_repos.append({"name": repo, "repository": f"googleapis/{repo}"})
return synth_repos
def list_repositories():
"""List all the available discovery versions grouped by api"""
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
repo = "googleapis/elixir-google-api"
api_config = gh.get_contents(repository=repo, path="config/apis.json")
api_names = set()
for api in json.loads(api_config):
api_names.add(api["name"])
return [{
"name": api,
"repository": "googleapis/elixir-google-api",
"branch-suffix": api.lower(),
"pr-title": f"Regenerate {api} client",
"args": [api],
"metadata-path": f"clients/{_underscore(api)}",
} for api in sorted(api_names)]
def list_repositories():
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
# Presently this only enumerates folders from the google-cloud-dotnet
# monorepo, under the autosynth directory.
repo = "googleapis/google-cloud-dotnet"
clients = gh.list_files(repo, "apis")
subdirs = [item["path"] for item in clients if item["type"] == "dir"]
# No hidden dirs
subdirs = [subdir for subdir in subdirs if not subdir.startswith(".")]
# Only subdirs that have synth.py files.
subdirs = [
subdir for subdir in subdirs if gh.check_for_file(repo, f"{subdir}/synth.py")
]
return [_config_for_subdir(repo, subdir) for subdir in subdirs]
def list_repositories():
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
# Presently this only enumerates folders from the google-cloud-php
# monorepo.
repo = "googleapis/google-cloud-php"
tree = gh.get_tree(repo)
subdirs = [item["path"] for item in tree["tree"] if item["type"] == "tree"]
# No hidden dirs
subdirs = [subdir for subdir in subdirs if not subdir.startswith(".")]
# Only subdirs that have synth.py files.
subdirs = [
subdir for subdir in subdirs
if gh.check_for_file(repo, f"{subdir}/synth.py")
]
return [{
"name": subdir.lower(),
"repository": repo,
"synth-path": subdir,
"branch-suffix": subdir.lower(),
} for subdir in subdirs]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config")
parser.add_argument("--github-token",
default=os.environ.get("GITHUB_TOKEN"))
parser.add_argument(
"--shard",
default=os.environ.get("MULTISYNTH_SHARD"),
help=
"From the list of repos, the shard number and shard count, separated by a forward slash. Examples:\n0/10\n1/2",
)
parser.add_argument("extra_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
config = load_config(args.config)
if config is None:
sys.exit("No configuration could be loaded.")
if args.shard:
config = select_shard(config, args.shard)
gh = github.GitHub(args.github_token)
base_log_path = find_base_log_path()
results = synthesize_libraries(config, gh, args.github_token,
args.extra_args[1:], base_log_path)
num_failures = len([result for result in results if result["error"]])
if num_failures > 0:
logger.error(f"Failed to synthesize {num_failures} job(s).")
failure_percent = 100 * num_failures / len(results)
if failure_percent < 12:
pass # It's most likely an issue with a few APIs.
else:
sys.exit(1) # Raise the attention of autosynth maintainers.
def list_repositories():
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
# Presently this only enumerates folders from the google-cloud-ruby
# monorepo.
repo = "googleapis/google-cloud-ruby"
default_branch = gh.get_default_branch(repo)
tree = gh.get_tree(repo, default_branch)
subdirs = [item["path"] for item in tree["tree"] if item["type"] == "tree"]
# No hidden dirs
subdirs = [subdir for subdir in subdirs if not subdir.startswith(".")]
# Only subdirs that have synth.py files.
subdirs = [subdir for subdir in subdirs if _check_bazel_synth(gh, repo, subdir)]
return [
{
"name": subdir.replace("google-cloud-", ""),
"repository": repo,
"synth-path": subdir,
"branch-suffix": subdir.replace("google-cloud-", ""),
}
for subdir in subdirs
]
def _inner_main(temp_dir: str) -> int:
"""
Returns:
int -- Number of commits committed to the repo.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--github-user", default=os.environ.get("GITHUB_USER"))
parser.add_argument("--github-email",
default=os.environ.get("GITHUB_EMAIL"))
parser.add_argument("--github-token",
default=os.environ.get("GITHUB_TOKEN"))
parser.add_argument("--repository",
default=os.environ.get("REPOSITORY"),
required=True)
parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH"))
parser.add_argument("--metadata-path",
default=os.environ.get("METADATA_PATH"))
parser.add_argument(
"--deprecated-execution",
default=False,
action="store_true",
help=
"If specified, execute synth.py directly instead of synthtool. This behavior is deprecated.",
)
parser.add_argument("--branch-suffix",
default=os.environ.get("BRANCH_SUFFIX", None))
parser.add_argument("--pr-title", default="")
parser.add_argument("extra_args", nargs=argparse.REMAINDER)
args = parser.parse_args()
gh = github.GitHub(args.github_token)
branch = "-".join(filter(None, ["autosynth", args.branch_suffix]))
pr_title = args.pr_title or (
f"[CHANGE ME] Re-generated {args.synth_path or ''} to pick up changes in "
f"the API or client library generator.")
change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh,
branch)
# capture logs for later
base_synth_log_path = pathlib.Path(
os.path.realpath("./logs")) / args.repository
if args.synth_path:
base_synth_log_path /= args.synth_path
logger.info(f"logs will be written to: {base_synth_log_path}")
working_repo_path = synthtool_git.clone(
f"https://github.com/{args.repository}.git")
try:
os.chdir(working_repo_path)
git.configure_git(args.github_user, args.github_email)
git.setup_branch(branch)
if args.synth_path:
os.chdir(args.synth_path)
metadata_path = os.path.join(args.metadata_path or "",
"synth.metadata")
flags = autosynth.flags.parse_flags()
# Override flags specified in synth.py with flags specified in environment vars.
for key in flags.keys():
env_value = os.environ.get(key, "")
if env_value:
flags[key] = False if env_value.lower(
) == "false" else env_value
metadata = load_metadata(metadata_path)
multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS]
multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS]
if (not multiple_commits and not multiple_prs) or not metadata:
if change_pusher.check_if_pr_already_exists(branch):
return 0
synth_log = Synthesizer(
metadata_path,
args.extra_args,
deprecated_execution=args.deprecated_execution,
).synthesize(base_synth_log_path)
if not has_changes():
logger.info("No changes. :)")
sys.exit(EXIT_CODE_SKIPPED)
git.commit_all_changes(pr_title)
change_pusher.push_changes(1, branch, pr_title, synth_log)
return 1
else:
if not multiple_prs and change_pusher.check_if_pr_already_exists(
branch):
return 0 # There's already an existing PR
# Enumerate the versions to loop over.
sources = metadata.get("sources", [])
source_versions = [
git_source.enumerate_versions_for_working_repo(
metadata_path, sources)
]
# Add supported source version types below:
source_versions.extend(
git_source.enumerate_versions(sources, pathlib.Path(temp_dir)))
# Prepare to call synthesize loop.
synthesizer = Synthesizer(
metadata_path,
args.extra_args,
args.deprecated_execution,
"synth.py",
)
x = SynthesizeLoopToolbox(
source_versions,
branch,
temp_dir,
metadata_path,
args.synth_path,
base_synth_log_path,
)
if not multiple_commits:
change_pusher = SquashingChangePusher(change_pusher)
# Call the loop.
commit_count = synthesize_loop(x, multiple_prs, change_pusher,
synthesizer)
if commit_count == 0:
logger.info("No changes. :)")
sys.exit(EXIT_CODE_SKIPPED)
return commit_count
finally:
if args.synth_path:
# We're generating code in a mono repo. The state left behind will
# probably be useful for generating the next API.
pass
else:
# We're generating a single API in a single repo, and using a different
# repo to generate the next API. So the next synth will not be able to
# use any of this state. Clean it up to avoid running out of disk space.
executor.run(["git", "clean", "-fdx"], cwd=working_repo_path)
def new_gh():
return github.GitHub(os.environ["GITHUB_TOKEN"])
def list_repositories():
gh = github.GitHub(os.environ["GITHUB_TOKEN"])
return _google_cloud_java_repos(gh) + _other_repos(gh)
