def replace(sources: ListOfPathsOrStrs,
before: str,
after: str,
flags: int = re.MULTILINE) -> int:
"""Replaces occurrences of before with after in all the given sources.
Returns:
The number of times the text was found and replaced across all files.
"""
expr = re.compile(before, flags=flags or 0)
paths = _filter_files(_expand_paths(sources, "."))
if not paths:
log.warning(f"No files were found in sources {sources} for replace()")
count_replaced = 0
for path in paths:
replaced = _replace_in_file(path, expr, after)
count_replaced += replaced
if replaced:
log.info(f"Replaced {before!r} in {path}.")
if not count_replaced:
log.warning(
f"No replacements made in {sources} for pattern {before}, maybe "
"replacement is not longer needed?")
return count_replaced
def _download_formatter(version: str, dest: Path) -> None:
log.info("Downloading java formatter")
url = JAR_DOWNLOAD_URL.format(version=version)
response = requests.get(url)
response.raise_for_status()
with open(dest, "wb") as fh:
fh.write(response.content)
def py_samples(*, root: PathOrStr = None, skip_readmes: bool = False) -> None:
"""
Find all samples projects and render templates.
Samples projects always have a 'requirements.txt' file and may also have
README.rst.in
Args:
root (Union[Path, str]): The samples directory root.
skip_readmes (bool): If true, do not generate readmes.
"""
in_client_library = Path("samples").exists() and Path("setup.py").exists()
if root is None:
if in_client_library:
root = "samples"
else:
root = "."
excludes = []
# todo(kolea2): temporary exclusion until samples are ready to be migrated to new format
excludes.append("README.md")
# TODO(busunkim): Readmegen is disabled as it requires installing the sample
# requirements in Synthtool. Sample Readmegen should be refactored to stop
# relying on the output of `python sample.py --help`
skip_readmes = True
if skip_readmes:
excludes.append("README.rst")
t = templates.TemplateGroup(SAMPLES_TEMPLATE_PATH, excludes=excludes)
t.env.globals["get_help"] = _get_help # for sample readmegen
for req in Path(root).glob("**/requirements.txt"):
sample_project_dir = req.parent
log.info(
f"Generating templates for samples project '{sample_project_dir}'")
excludes = ["**/*tmpl*"] # .tmpl. files are partial templates
sample_readme_metadata: Dict[str, Any] = {}
if not skip_readmes:
sample_readme_metadata = _get_sample_readme_metadata(
sample_project_dir)
# Don't generate readme if there's no metadata
if sample_readme_metadata == {}:
excludes.append("**/README.rst")
if Path(sample_project_dir / "noxfile_config.py").exists():
# Don't overwrite existing noxfile configs
excludes.append("**/noxfile_config.py")
result = t.render(subdir=sample_project_dir, **sample_readme_metadata)
_tracked_paths.add(result)
s.copy([result], excludes=excludes)
def format_code(path: str, version: str = DEFAULT_FORMAT_VERSION) -> None:
"""
Runs the google-java-format jar against all .java files found within the
provided path.
"""
jar_name = f"google-java-format-{version}.jar"
jar = cache.get_cache_dir() / jar_name
if not jar.exists():
_download_formatter(version, jar)
# Find all .java files in path and run the formatter on them
files = list(glob.iglob(os.path.join(path, "**/*.java"), recursive=True))
# Run the formatter as a jar file
log.info("Running java formatter on {} files".format(len(files)))
shell.run(["java", "-jar", str(jar), "--replace"] + files)
def _remove_obsolete_files(old_metadata):
"""Remove obsolete files from the file system.
Call add_new_files() before this function or it will remove all generated
files.
Parameters:
old_metadata: old metadata loaded from a call to read_or_empty().
"""
old_files = set([new_file.path for new_file in old_metadata.new_files])
new_files = set([new_file.path for new_file in _metadata.new_files])
obsolete_files = old_files - new_files
for file_path in git_ignore(obsolete_files):
try:
log.info(f"Removing obsolete file {file_path}...")
os.unlink(file_path)
except FileNotFoundError:
pass # Already deleted. That's OK.
def py_samples(*, root: PathOrStr = None, skip_readmes: bool = False) -> None:
"""
Find all samples projects and render templates.
Samples projects always have a 'requirements.txt' file and may also have
README.rst.in
Args:
root (Union[Path, str]): The samples directory root.
skip_readmes (bool): If true, do not generate readmes.
"""
in_client_library = Path("samples").exists() and Path("setup.py").exists()
if root is None:
if in_client_library:
root = "samples"
else:
root = "."
excludes = []
if skip_readmes:
excludes.append("README.rst")
t = templates.TemplateGroup(SAMPLES_TEMPLATE_PATH, excludes=excludes)
t.env.globals["get_help"] = _get_help # for sample readmegen
for req in Path(root).glob("**/requirements.txt"):
sample_project_dir = req.parent
log.info(f"Generating templates for samples project '{sample_project_dir}'")
excludes = ["**/*tmpl*"] # .tmpl. files are partial templates
sample_readme_metadata: Dict[str, Any] = {}
if not skip_readmes:
sample_readme_metadata = _get_sample_readme_metadata(sample_project_dir)
# Don't generate readme if there's no metadata
if sample_readme_metadata == {}:
excludes.append("**/README.rst")
if Path(sample_project_dir / "noxfile_config.py").exists():
# Don't overwrite existing noxfile configs
excludes.append("**/noxfile_config.py")
result = t.render(subdir=sample_project_dir, **sample_readme_metadata)
_tracked_paths.add(result)
s.copy([result], excludes=excludes)
def generate_service(disco: str):
m = re.search(VERSION_REGEX, disco)
name = m.group(1)
version = m.group(2)
template = TEMPLATE_VERSIONS[-1] # Generate for latest version
log.info(f"Generating {name} {version} ({template}).")
output_dir = repository / ".cache" / name / version
input_file = discovery / "discoveries" / disco
command = (
f"python2 -m googleapis.codegen --output_dir={output_dir}" +
f" --input={input_file} --language=php --language_variant={template}" +
f" --package_path=api/services")
shell.run(f"mkdir -p {output_dir}".split(), cwd=repository / "generator")
shell.run(command.split(), cwd=repository, hide_output=False)
s.copy(output_dir, f"src")
def replace(sources: ListOfPathsOrStrs,
before: str,
after: str,
flags: int = re.MULTILINE):
"""Replaces occurrences of before with after in all the given sources."""
expr = re.compile(before, flags=flags or 0)
paths = _filter_files(_expand_paths(sources, "."))
if not paths:
log.warning(f"No files were found in sources {sources} for replace()")
any_replaced = False
for path in paths:
replaced = _replace_in_file(path, expr, after)
any_replaced = any_replaced or replaced
if replaced:
log.info(f"Replaced {before!r} in {path}.")
if not any_replaced:
log.warning(
f"No replacements made in {sources} for pattern {before}, maybe "
"replacement is not longer needed?")
def generate_service(disco: str):
m = re.search(VERSION_REGEX, disco)
if m is None:
log.info(f"Skipping {disco}.")
return
name = m.group(1)
version = m.group(2)
log.info(f"Generating {name} {version}.")
library_name = f"google-api-services-{name}"
output_dir = repository / ".cache" / library_name / version
input_file = discovery / "discoveries" / disco
for template in TEMPLATE_VERSIONS:
log.info(f"\t{template}")
command = (
f"python2 -m googleapis.codegen --output_dir={output_dir}" +
f" --input={input_file} --language=java --language_variant={template}"
+ f" --package_path=api/services")
shell.run(f"mkdir -p {output_dir}".split(),
cwd=repository / "generator")
shell.run(command.split(), cwd=repository, hide_output=False)
s.copy(output_dir, f"clients/{library_name}/{version}/{template}")
resource_dir = (repository / "clients" / library_name / version /
template / "resources")
shell.run(f"mkdir -p {resource_dir}".split())
write_discovery_file(input_file, resource_dir / path.basename(disco))
# write the metadata file
latest_version = TEMPLATE_VERSIONS[-1]
metadata = maven_metadata(
str(repository / "clients" / library_name / version / latest_version /
"pom.xml"))
write_metadata_file(library_name, version, metadata)
# copy the latest README to the main service location
shutil.copy(
repository / "clients" / library_name / version / latest_version /
"README.md",
repository / "clients" / library_name / version / "README.md",
)
def _include_samples(
self,
language: str,
version: str,
genfiles: Path,
googleapis_service_dir: Path,
samples_root_dir: Path = None,
samples_resources_dir: Path = None,
):
"""Include code samples and supporting resources in generated output.
Resulting directory structure in generated output:
samples/
├── resources
│ ├── example_text_file.txt
│ └── example_data.csv
└── v1/
├── sample_one.py
├── sample_two.py
└── test/
├── samples.manifest.yaml
├── sample_one.test.yaml
└── sample_two.test.yaml
Samples are included in the genfiles output of the generator.
Sample tests are defined in googleapis:
{service}/{version}/samples/test/*.test.yaml
Sample resources are declared in {service}/sample_resources.yaml
which includes a list of files with public gs:// URIs for download.
Sample resources are files needed to run code samples or system tests.
Synth keeps resources in sync by always pulling down the latest version.
It is recommended to store resources in the `cloud-samples-data` bucket.
Sample manifest is a generated file which defines invocation commands
for each code sample (used by sample-tester to invoke samples).
"""
if samples_root_dir is None:
samples_root_dir = genfiles / "samples"
if samples_resources_dir is None:
samples_resources_dir = samples_root_dir / "resources"
samples_version_dir = samples_root_dir / version
# Some languages capitalize their `V` prefix for version numbers
if not samples_version_dir.is_dir():
samples_version_dir = samples_root_dir / version.capitalize()
# Do not proceed if genfiles does not include samples/{version} dir.
if not samples_version_dir.is_dir():
return None
samples_test_dir = samples_version_dir / "test"
samples_manifest_yaml = samples_test_dir / "samples.manifest.yaml"
googleapis_samples_dir = googleapis_service_dir / version / "samples"
googleapis_resources_yaml = googleapis_service_dir / "sample_resources.yaml"
# Copy sample tests from googleapis {service}/{version}/samples/*.test.yaml
# into generated output as samples/{version}/test/*.test.yaml
test_files = googleapis_samples_dir.glob("**/*.test.yaml")
os.makedirs(samples_test_dir, exist_ok=True)
for i in test_files:
log.debug(f"Copy: {i} to {samples_test_dir / i.name}")
shutil.copyfile(i, samples_test_dir / i.name)
# Download sample resources from sample_resources.yaml storage URIs.
#
# sample_resources:
# - uri: gs://bucket/the/file/path.csv
# description: Description of this resource
#
# Code follows happy path. An error is desirable if YAML is invalid.
if googleapis_resources_yaml.is_file():
with open(googleapis_resources_yaml, "r") as f:
resources_data = yaml.load(f, Loader=yaml.SafeLoader)
resource_list = resources_data.get("sample_resources")
for resource in resource_list:
uri = resource.get("uri")
if uri.startswith("gs://"):
uri = uri.replace("gs://",
"https://storage.googleapis.com/")
response = requests.get(uri, allow_redirects=True)
download_path = samples_resources_dir / os.path.basename(uri)
os.makedirs(samples_resources_dir, exist_ok=True)
log.debug(f"Download {uri} to {download_path}")
with open(download_path, "wb") as output: # type: ignore
output.write(response.content)
# Generate manifest file at samples/{version}/test/samples.manifest.yaml
# Includes a reference to every sample (via its "region tag" identifier)
# along with structured instructions on how to invoke that code sample.
relative_manifest_path = str(
samples_manifest_yaml.relative_to(samples_root_dir))
LANGUAGE_EXECUTABLES = {
"nodejs": "node",
"php": "php",
"python": "python3",
"ruby": "bundle exec ruby",
}
if language not in LANGUAGE_EXECUTABLES:
log.info("skipping manifest gen")
return None
manifest_arguments = [
"gen-manifest",
f"--env={language}",
f"--bin={LANGUAGE_EXECUTABLES[language]}",
f"--output={relative_manifest_path}",
"--chdir={@manifest_dir}/../..",
]
for code_sample in samples_version_dir.glob("*"):
sample_path = str(code_sample.relative_to(samples_root_dir))
if os.path.isfile(code_sample):
manifest_arguments.append(sample_path)
try:
log.debug(f"Writing samples manifest {manifest_arguments}")
shell.run(manifest_arguments, cwd=samples_root_dir)
except (subprocess.CalledProcessError, FileNotFoundError):
log.warning(
"gen-manifest failed (sample-tester may not be installed)")
