def add_sample_client_destination():
metadata.add_client_destination(
source="source",
api_name="api",
api_version="v1",
language="py",
generator="gen",
config="config",
)
def test_add_client_destination():
metadata.reset()
metadata.add_client_destination(
source="source",
api_name="api",
api_version="v1",
language="py",
generator="gen",
config="config",
)
current = metadata.get()
assert current.destinations[0].client.source == "source"
assert current.destinations[0].client.api_name == "api"
assert current.destinations[0].client.api_version == "v1"
assert current.destinations[0].client.language == "py"
assert current.destinations[0].client.generator == "gen"
assert current.destinations[0].client.config == "config"
def _generate_code(
self,
service: str,
version: str,
language: str,
*,
private: bool = False,
proto_path: Union[str, Path] = None,
extra_proto_files: List[str] = [],
output_dir: Union[str, Path] = None,
generator_version: str = "latest",
generator_args: Mapping[str, str] = None,
):
# Determine which googleapis repo to use
if not private:
googleapis = self._clone_googleapis()
else:
googleapis = self._clone_googleapis_private()
# Sanity check: We should have a googleapis repo; if we do not,
# something went wrong, and we should abort.
if googleapis is None:
raise RuntimeError(
f"Unable to generate {service}, the googleapis repository"
"is unavailable.")
# Pull the code generator for the requested language.
# If a code generator version was specified, honor that.
log.debug(
f"Pulling Docker image: gapic-generator-{language}:{generator_version}"
)
shell.run(
[
"docker",
"pull",
f"gcr.io/gapic-images/gapic-generator-{language}:{generator_version}",
],
hide_output=False,
)
# Determine where the protos we are generating actually live.
# We can sometimes (but not always) determine this from the service
# and version; in other cases, the user must provide it outright.
if proto_path:
proto_path = Path(proto_path)
if proto_path.is_absolute():
proto_path = proto_path.relative_to("/")
else:
proto_path = Path("google/cloud") / service / version
# Sanity check: Do we have protos where we think we should?
if not (googleapis / proto_path).exists():
raise FileNotFoundError(
f"Unable to find directory for protos: {(googleapis / proto_path)}."
)
if not tuple((googleapis / proto_path).glob("*.proto")):
raise FileNotFoundError(
f"Directory {(googleapis / proto_path)} exists, but no protos found."
)
# Ensure the desired output directory exists.
# If none was provided, create a temporary directory.
if not output_dir:
output_dir = tempfile.mkdtemp()
output_dir = Path(output_dir).resolve()
# The time has come, the walrus said, to talk of actually running
# the code generator.
sep = os.path.sep
# try to figure out user ID and stay compatible.
# If there is no `os.getuid()`, fallback to `getpass.getuser()`
getuid = getattr(os, "getuid", None)
if getuid:
user = str(getuid())
else:
user = getpass.getuser()
docker_run_args = [
"docker",
"run",
"--mount",
f"type=bind,source={googleapis / proto_path}{sep},destination={Path('/in') / proto_path}{sep},readonly",
"--mount",
f"type=bind,source={output_dir}{sep},destination={Path('/out')}{sep}",
"--rm",
"--user",
user,
]
# Process extra proto files, e.g. google/cloud/common_resources.proto,
# if they are required by this API.
# First, bind mount all the extra proto files into the container.
for proto in extra_proto_files:
source_proto = googleapis / Path(proto)
if not source_proto.exists():
raise FileNotFoundError(
f"Unable to find extra proto file: {source_proto}.")
docker_run_args.extend([
"--mount",
f"type=bind,source={source_proto},destination={Path('/in') / proto},readonly",
])
docker_run_args.append(
f"gcr.io/gapic-images/gapic-generator-{language}:{generator_version}"
)
# Populate any additional CLI arguments provided for Docker.
if generator_args:
for key, value in generator_args.items():
docker_run_args.append(f"--{key}")
docker_run_args.append(value)
log.debug(f"Generating code for: {proto_path}.")
shell.run(docker_run_args)
# Sanity check: Does the output location have code in it?
# If not, complain.
if not tuple(output_dir.iterdir()):
raise RuntimeError(
f"Code generation seemed to succeed, but {output_dir} is empty."
)
# Huzzah, it worked.
log.success(f"Generated code into {output_dir}.")
# Record this in the synthtool metadata.
metadata.add_client_destination(
source="googleapis" if not private else "googleapis-private",
api_name=service,
api_version=version,
language=language,
generator=f"gapic-generator-{language}",
)
_tracked_paths.add(output_dir)
return output_dir
def _generate_code(
self,
service,
version,
language,
config_path=None,
artman_output_name=None,
private=False,
include_protos=False,
generator_args=None,
):
# map the language to the artman argument and subdir of genfiles
GENERATE_FLAG_LANGUAGE = {
"python": ("python_gapic", "python"),
"nodejs": ("nodejs_gapic", "js"),
"ruby": ("ruby_gapic", "ruby"),
"php": ("php_gapic", "php"),
"java": ("java_gapic", "java"),
}
if language not in GENERATE_FLAG_LANGUAGE:
raise ValueError("provided language unsupported")
gapic_language_arg, gen_language = GENERATE_FLAG_LANGUAGE[language]
# Determine which googleapis repo to use
if not private:
googleapis = self._clone_googleapis()
else:
googleapis = self._clone_googleapis_private()
if googleapis is None:
raise RuntimeError(
f"Unable to generate {config_path}, the googleapis repository"
"is unavailable.")
generator_dir = LOCAL_GENERATOR
if generator_dir is not None:
log.debug(f"Using local generator at {generator_dir}")
# Run the code generator.
# $ artman --config path/to/artman_api.yaml generate python_gapic
if config_path is None:
config_path = (Path("google/cloud") / service /
f"artman_{service}_{version}.yaml")
elif Path(config_path).is_absolute():
config_path = Path(config_path).relative_to("/")
else:
config_path = Path("google/cloud") / service / Path(config_path)
if not (googleapis / config_path).exists():
raise FileNotFoundError(
f"Unable to find configuration yaml file: {(googleapis / config_path)}."
)
log.debug(f"Running generator for {config_path}.")
output_root = self._artman.run(
f"googleapis/artman:{artman.ARTMAN_VERSION}",
googleapis,
config_path,
gapic_language_arg,
generator_dir=generator_dir,
generator_args=generator_args,
)
# Expect the output to be in the artman-genfiles directory.
# example: /artman-genfiles/python/speech-v1
if artman_output_name is None:
artman_output_name = f"{service}-{version}"
genfiles = output_root / gen_language / artman_output_name
if not genfiles.exists():
raise FileNotFoundError(
f"Unable to find generated output of artman: {genfiles}.")
log.success(f"Generated code into {genfiles}.")
# Get the *.protos files and put them in a protos dir in the output
if include_protos:
import shutil
source_dir = googleapis / config_path.parent / version
proto_files = source_dir.glob("**/*.proto")
# By default, put the protos at the root in a folder named 'protos'.
# Specific languages can be cased here to put them in a more language
# appropriate place.
proto_output_path = genfiles / "protos"
if language == "python":
# place protos alongsize the *_pb2.py files
proto_output_path = genfiles / f"google/cloud/{service}_{version}/proto"
os.makedirs(proto_output_path, exist_ok=True)
for i in proto_files:
log.debug(f"Copy: {i} to {proto_output_path / i.name}")
shutil.copyfile(i, proto_output_path / i.name)
log.success(f"Placed proto files into {proto_output_path}.")
metadata.add_client_destination(
source="googleapis" if not private else "googleapis-private",
api_name=service,
api_version=version,
language=language,
generator="gapic",
config=str(config_path),
)
_tracked_paths.add(genfiles)
return genfiles
def _generate_code(
self,
service: str,
version: str,
language: str,
*,
private: bool = False,
discogapic: bool = False,
proto_path: Union[str, Path] = None,
output_dir: Union[str, Path] = None,
bazel_target: str = None,
include_protos: bool = False,
proto_output_path: Union[str, Path] = None,
tar_strip_components: int = 1,
):
# Determine which googleapis repo to use
if discogapic:
api_definitions_repo = self._clone_discovery_artifact_manager()
api_definitions_repo_name = "discovery-artifact-manager"
elif private:
api_definitions_repo = self._clone_googleapis_private()
api_definitions_repo_name = "googleapis_private"
else:
api_definitions_repo = self._clone_googleapis()
api_definitions_repo_name = "googleapis"
# Sanity check: We should have a googleapis repo; if we do not,
# something went wrong, and we should abort.
if not api_definitions_repo:
raise RuntimeError(
f"Unable to generate {service}, the sources repository repository"
"is unavailable."
)
# Calculate proto_path if necessary.
if not bazel_target or include_protos:
# If bazel_target is not specified explicitly, we will need
# proto_path to calculate it. If include_protos is True,
# we will need the proto_path to copy the protos.
if not proto_path:
if bazel_target:
# Calculate proto_path from the full bazel target, which is
# in the format "//proto_path:target_name
proto_path = bazel_target.split(":")[0][2:]
else:
# If bazel_target is not specified, assume the protos are
# simply under google/cloud, where the most of the protos
# usually are.
proto_path = f"google/cloud/{service}/{version}"
protos = Path(proto_path)
if protos.is_absolute():
protos = protos.relative_to("/")
# Determine bazel target based on per-language patterns
# Java: google-cloud-{{assembly_name}}-{{version}}-java
# Go: gapi-cloud-{{assembly_name}}-{{version}}-go
# Python: {{assembly_name}}-{{version}}-py
# PHP: google-cloud-{{assembly_name}}-{{version}}-php
# Node.js: {{assembly_name}}-{{version}}-nodejs
# Ruby: google-cloud-{{assembly_name}}-{{version}}-ruby
# C#: google-cloud-{{assembly_name}}-{{version}}-csharp
if not bazel_target:
# Determine where the protos we are generating actually live.
# We can sometimes (but not always) determine this from the service
# and version; in other cases, the user must provide it outright.
parts = list(protos.parts)
while len(parts) > 0 and parts[0] != "google":
parts.pop(0)
if len(parts) == 0:
raise RuntimeError(
f"Cannot determine bazel_target from proto_path {protos}."
"Please set bazel_target explicitly."
)
if language == "python":
suffix = f"{service}-{version}-py"
elif language == "nodejs":
suffix = f"{service}-{version}-nodejs"
elif language == "go":
suffix = f"gapi-{'-'.join(parts[1:])}-go"
else:
suffix = f"{'-'.join(parts)}-{language}"
bazel_target = f"//{os.path.sep.join(parts)}:{suffix}"
# Sanity check: Do we have protos where we think we should?
if not (api_definitions_repo / protos).exists():
raise FileNotFoundError(
f"Unable to find directory for protos: {(api_definitions_repo / protos)}."
)
if not tuple((api_definitions_repo / protos).glob("*.proto")):
raise FileNotFoundError(
f"Directory {(api_definitions_repo / protos)} exists, but no protos found."
)
if not (api_definitions_repo / protos / "BUILD.bazel"):
raise FileNotFoundError(
f"File {(api_definitions_repo / protos / 'BUILD.bazel')} does not exist."
)
# Ensure the desired output directory exists.
# If none was provided, create a temporary directory.
if not output_dir:
output_dir = tempfile.mkdtemp()
output_dir = Path(output_dir).resolve()
# Let's build some stuff now.
cwd = os.getcwd()
os.chdir(str(api_definitions_repo))
bazel_run_args = [
"bazel",
"--max_idle_secs=240",
"build",
bazel_target,
]
logger.debug(f"Generating code for: {bazel_target}.")
shell.run(bazel_run_args)
# We've got tar file!
# its location: bazel-bin/google/cloud/language/v1/language-v1-nodejs.tar.gz
# bazel_target: //google/cloud/language/v1:language-v1-nodejs
tar_file = (
f"bazel-bin{os.path.sep}{bazel_target[2:].replace(':', os.path.sep)}.tar.gz"
)
tar_run_args = [
"tar",
"-C",
str(output_dir),
f"--strip-components={tar_strip_components}",
"-xzf",
tar_file,
]
shell.run(tar_run_args)
# Get the *.protos files and put them in a protos dir in the output
if include_protos:
proto_files = protos.glob("**/*.proto")
# By default, put the protos at the root in a folder named 'protos'.
# Specific languages can be cased here to put them in a more language
# appropriate place.
if not proto_output_path:
proto_output_path = output_dir / "protos"
if language == "python":
# place protos alongsize the *_pb2.py files
proto_output_path = (
output_dir / f"google/cloud/{service}_{version}/proto"
)
else:
proto_output_path = Path(output_dir / proto_output_path)
os.makedirs(proto_output_path, exist_ok=True)
for i in proto_files:
logger.debug(f"Copy: {i} to {proto_output_path / i.name}")
shutil.copyfile(i, proto_output_path / i.name)
logger.success(f"Placed proto files into {proto_output_path}.")
os.chdir(cwd)
# Sanity check: Does the output location have code in it?
# If not, complain.
if not tuple(output_dir.iterdir()):
raise RuntimeError(
f"Code generation seemed to succeed, but {output_dir} is empty."
)
# Huzzah, it worked.
logger.success(f"Generated code into {output_dir}.")
# Record this in the synthtool metadata.
metadata.add_client_destination(
source=api_definitions_repo_name,
api_name=service,
api_version=version,
language=language,
generator="bazel",
)
_tracked_paths.add(output_dir)
return output_dir
def _generate_code(
self,
service: str,
version: str,
language: str,
*,
private: bool = False,
proto_path: Union[str, Path] = None,
output_dir: Union[str, Path] = None,
bazel_target: str = None,
):
# Determine which googleapis repo to use
if not private:
googleapis = self._clone_googleapis()
else:
googleapis = self._clone_googleapis_private()
# Sanity check: We should have a googleapis repo; if we do not,
# something went wrong, and we should abort.
if googleapis is None:
raise RuntimeError(
f"Unable to generate {service}, the googleapis repository"
"is unavailable."
)
# Determine where the protos we are generating actually live.
# We can sometimes (but not always) determine this from the service
# and version; in other cases, the user must provide it outright.
if proto_path:
proto_path = Path(proto_path)
if proto_path.is_absolute():
proto_path = proto_path.relative_to("/")
else:
proto_path = Path("google/cloud") / service / version
# Determine bazel target based on per-language patterns
# Java: google-cloud-{{assembly_name}}-{{version}}-java
# Go: gapi-cloud-{{assembly_name}}-{{version}}-go
# Python: {{assembly_name}}-{{version}}-py
# PHP: google-cloud-{{assembly_name}}-{{version}}-php
# Node.js: {{assembly_name}}-{{version}}-nodejs
# Ruby: google-cloud-{{assembly_name}}-{{version}}-ruby
# C#: google-cloud-{{assembly_name}}-{{version}}-csharp
if bazel_target is None:
parts = list(proto_path.parts)
while len(parts) > 0 and parts[0] != "google":
parts.pop(0)
if len(parts) == 0:
raise RuntimeError(
f"Cannot determine bazel_target from proto_path {proto_path}."
"Please set bazel_target explicitly."
)
if language == "python":
suffix = f"{service}-{version}-py"
elif language == "nodejs":
suffix = f"{service}-{version}-nodejs"
elif language == "go":
suffix = f"gapi-{'-'.join(parts[1:])}-go"
else:
suffix = f"{'-'.join(parts)}-{language}"
bazel_target = f"//{os.path.sep.join(parts)}:{suffix}"
# Sanity check: Do we have protos where we think we should?
if not (googleapis / proto_path).exists():
raise FileNotFoundError(
f"Unable to find directory for protos: {(googleapis / proto_path)}."
)
if not tuple((googleapis / proto_path).glob("*.proto")):
raise FileNotFoundError(
f"Directory {(googleapis / proto_path)} exists, but no protos found."
)
if not (googleapis / proto_path / "BUILD.bazel"):
raise FileNotFoundError(
f"File {(googleapis / proto_path / 'BUILD.bazel')} does not exist."
)
# Ensure the desired output directory exists.
# If none was provided, create a temporary directory.
if not output_dir:
output_dir = tempfile.mkdtemp()
output_dir = Path(output_dir).resolve()
# Let's build some stuff now.
cwd = os.getcwd()
os.chdir(str(googleapis))
bazel_run_args = ["bazel", "build", bazel_target]
log.debug(f"Generating code for: {proto_path}.")
shell.run(bazel_run_args)
# We've got tar file!
# its location: bazel-bin/google/cloud/language/v1/language-v1-nodejs.tar.gz
# bazel_target: //google/cloud/language/v1:language-v1-nodejs
tar_file = (
f"bazel-bin{os.path.sep}{bazel_target[2:].replace(':', os.path.sep)}.tar.gz"
)
tar_run_args = [
"tar",
"-C",
str(output_dir),
"--strip-components=1",
"-xzf",
tar_file,
]
shell.run(tar_run_args)
os.chdir(cwd)
# Sanity check: Does the output location have code in it?
# If not, complain.
if not tuple(output_dir.iterdir()):
raise RuntimeError(
f"Code generation seemed to succeed, but {output_dir} is empty."
)
# Huzzah, it worked.
log.success(f"Generated code into {output_dir}.")
# Record this in the synthtool metadata.
metadata.add_client_destination(
source="googleapis" if not private else "googleapis-private",
api_name=service,
api_version=version,
language=language,
generator="bazel",
)
_tracked_paths.add(output_dir)
return output_dir
def _generate_code(
self,
service: str,
version: str,
language: str,
*,
private: bool = False,
proto_path: Union[str, Path] = None,
output_dir: Union[str, Path] = None,
generator_version: str = "latest",
generator_args: Mapping[str, str] = None,
):
# Determine which googleapis repo to use
if not private:
googleapis = self._clone_googleapis()
else:
googleapis = self._clone_googleapis_private()
# Sanity check: We should have a googleapis repo; if we do not,
# something went wrong, and we should abort.
if googleapis is None:
raise RuntimeError(
f"Unable to generate {service}, the googleapis repository"
"is unavailable.")
# Pull the code generator for the requested language.
# If a code generator version was specified, honor that.
log.debug(
"Pulling Docker image: gapic-generator-{language}:{generator_version}"
)
shell.run(
[
"docker",
"pull",
f"gcr.io/gapic-images/gapic-generator-{language}:{generator_version}",
],
hide_output=False,
)
# Determine where the protos we are generating actually live.
# We can sometimes (but not always) determine this from the service
# and version; in other cases, the user must provide it outright.
if proto_path:
proto_path = Path(proto_path)
if proto_path.is_absolute():
proto_path = proto_path.relative_to("/")
else:
proto_path = Path("google/cloud") / service / version
# Sanity check: Do we have protos where we think we should?
if not (googleapis / proto_path).exists():
raise FileNotFoundError(
f"Unable to find directory for protos: {(googleapis / proto_path)}."
)
if not tuple((googleapis / proto_path).glob("*.proto")):
raise FileNotFoundError(
f"Directory {(googleapis / proto_path)} exists, but no protos found."
)
# Ensure the desired output directory exists.
# If none was provided, create a temporary directory.
if not output_dir:
output_dir = tempfile.mkdtemp()
output_dir = Path(output_dir).resolve()
# The time has come, the walrus said, to talk of actually running
# the code generator.
log.debug(f"Generating code for: {proto_path}.")
sep = os.path.sep
shell.run([
"docker",
"run",
"--mount",
f"type=bind,source={googleapis / proto_path}{sep},destination={Path('/in') / proto_path}{sep},readonly",
"--mount",
f"type=bind,source={output_dir}{sep},destination={Path('/out')}{sep}",
"--rm",
"--user",
str(os.getuid()),
f"gcr.io/gapic-images/gapic-generator-{language}",
])
# Sanity check: Does the output location have code in it?
# If not, complain.
if not tuple(output_dir.iterdir()):
raise RuntimeError(
f"Code generation seemed to succeed, but {output_dir} is empty."
)
# Huzzah, it worked.
log.success(f"Generated code into {output_dir}.")
# Record this in the synthtool metadata.
metadata.add_client_destination(
source="googleapis" if not private else "googleapis-private",
api_name=service,
api_version=version,
language=language,
generator=f"gapic-generator-{language}",
)
_tracked_paths.add(output_dir)
return output_dir
