def _remove_obsolete_files(old_metadata):
"""Remove obsolete files from the file system.
Call add_new_files() before this function or it will remove all generated
files.
Parameters:
old_metadata: old metadata loaded from a call to read_or_empty().
"""
old_files = set(old_metadata.generated_files)
new_files = set(_metadata.generated_files)
excluded_patterns = set([pattern for pattern in _excluded_patterns])
obsolete_files = old_files - new_files
for file_path in git_ignore(obsolete_files):
try:
matched_pattern = False
for pattern in excluded_patterns:
if fnmatch.fnmatch(file_path, pattern):
matched_pattern = True
break
if matched_pattern:
logger.info(
f"Leaving obsolete file {file_path} because it matched excluded pattern {pattern} during copy."
)
else:
logger.info(f"Removing obsolete file {file_path}...")
os.unlink(file_path)
except FileNotFoundError:
pass # Already deleted. That's OK.
def _download_formatter(version: str, dest: Path) -> None:
logger.info("Downloading java formatter")
url = JAR_DOWNLOAD_URL.format(version=version)
response = requests.get(url)
response.raise_for_status()
with open(dest, "wb") as fh:
fh.write(response.content)
def replace(
sources: ListOfPathsOrStrs, before: str, after: str, flags: int = re.MULTILINE
) -> int:
"""Replaces occurrences of before with after in all the given sources.
Returns:
The number of times the text was found and replaced across all files.
"""
expr = re.compile(before, flags=flags or 0)
paths = _filter_files(_expand_paths(sources, "."))
if not paths:
logger.warning(f"No files were found in sources {sources} for replace()")
count_replaced = 0
for path in paths:
replaced = _replace_in_file(path, expr, after)
count_replaced += replaced
if replaced:
logger.info(f"Replaced {before!r} in {path}.")
if not count_replaced:
logger.warning(
f"No replacements made in {sources} for pattern {before}, maybe "
"replacement is no longer needed?"
)
return count_replaced
def owlbot_copy_version(
src: Path,
dest: Path,
copy_excludes: typing.Optional[typing.List[str]] = None,
) -> None:
"""Copies files from a version subdirectory.
"""
logger.debug("owlbot_copy_version called from %s to %s", src, dest)
if copy_excludes is None:
copy_excludes = DEFAULT_COPY_EXCLUDES
# detect the version string for later use
src_dir = src / "src"
entries = os.scandir(src_dir)
if not entries:
logger.info("there is no src directory '%s' to copy", src_dir)
return
version_string = os.path.basename(os.path.basename(next(entries)))
logger.debug("version_string detected: %s", version_string)
# copy all src including partial veneer classes
s.move([src / "src"], dest / "src", merge=_merge, excludes=copy_excludes)
# copy tests
s.move([src / "tests"],
dest / "tests",
merge=_merge,
excludes=copy_excludes)
# detect the directory containing proto generated PHP source and metadata.
proto_src = src / "proto/src"
entries = os.scandir(proto_src)
proto_dir = None
metadata_dir = None
if not entries:
logger.info("there is no proto generated src directory to copy: %s",
proto_src)
return
for entry in entries:
if os.path.basename(entry.path) == METADATA_DIR:
metadata_dir = _find_copy_target(
Path(entry.path).resolve(), version_string)
else:
proto_dir = _find_copy_target(
Path(entry.path).resolve(), version_string)
# copy proto files
if isinstance(proto_dir, Path):
logger.debug("proto_dir detected: %s", proto_dir)
s.move([proto_dir], dest / "src", merge=_merge, excludes=copy_excludes)
# copy metadata files
if isinstance(metadata_dir, Path):
logger.debug("metadata_dir detected: %s", metadata_dir)
s.move([metadata_dir],
dest / "metadata",
merge=_merge,
excludes=copy_excludes)
def format_code(path: str, times: int = 2) -> None:
"""
Runs the google-java-format jar against all .java files found within the
provided path.
"""
# Find all .java files in path and run the formatter on them
files = list(glob.iglob(os.path.join(path, "**/*.java"), recursive=True))
# Run the formatter as a jar file
logger.info("Running java formatter on {} files".format(len(files)))
formatter_binary = sys.argv[2]
for _ in range(times):
shell.run([formatter_binary, "--replace"] + files)
def generate_index_ts(versions: List[str], default_version: str) -> None:
"""
generate src/index.ts to export the client name and versions in the client library.
Args:
versions: the list of versions, like: ['v1', 'v1beta1', ...]
default_version: a stable version provided by API producer. It must exist in argument versions.
Return:
True/False: return true if successfully generate src/index.ts, vice versa.
"""
# sanitizer the input arguments
if len(versions) < 1:
err_msg = (
"List of version can't be empty, it must contain default version at least."
)
logger.error(err_msg)
raise AttributeError(err_msg)
if default_version not in versions:
err_msg = f"Version {versions} must contain default version {default_version}."
logger.error(err_msg)
raise AttributeError(err_msg)
# To make sure the output is always deterministic.
versions = sorted(versions)
# compose default version's index.ts file path
versioned_index_ts_path = Path("src") / default_version / "index.ts"
clients = extract_clients(versioned_index_ts_path)
if not clients:
err_msg = f"No client is exported in the default version's({default_version}) index.ts ."
logger.error(err_msg)
raise AttributeError(err_msg)
# compose template directory
template_path = (
Path(__file__).parent.parent / "gcp" / "templates" / "node_split_library"
)
template_loader = FileSystemLoader(searchpath=str(template_path))
template_env = Environment(loader=template_loader, keep_trailing_newline=True)
TEMPLATE_FILE = "index.ts.j2"
index_template = template_env.get_template(TEMPLATE_FILE)
# render index.ts content
output_text = index_template.render(
versions=versions, default_version=default_version, clients=clients
)
with open("src/index.ts", "w") as fh:
fh.write(output_text)
logger.info("successfully generate `src/index.ts`")
def owlbot_main(
src: Path,
dest: Path,
copy_excludes: typing.Optional[typing.List[str]] = None,
patch_func: typing.Callable[[], None] = owlbot_patch,
) -> None:
"""Copies files from generated tree.
"""
entries = os.scandir(src)
if not entries:
logger.info("there is no version subdirectory to copy")
return
for entry in entries:
if entry.is_dir():
version_src = Path(entry.path).resolve()
owlbot_copy_version(version_src, dest, copy_excludes)
with pushd(dest):
patch_func()
def _remove_obsolete_files(old_metadata):
"""Remove obsolete files from the file system.
Call add_new_files() before this function or it will remove all generated
files.
Parameters:
old_metadata: old metadata loaded from a call to read_or_empty().
"""
old_files = set(old_metadata.generated_files)
new_files = set(_metadata.generated_files)
obsolete_files = old_files - new_files
for file_path in git_ignore(obsolete_files):
try:
logger.info(f"Removing obsolete file {file_path}...")
os.unlink(file_path)
except FileNotFoundError:
pass # Already deleted. That's OK.
def format_code(path: str,
version: str = DEFAULT_FORMAT_VERSION,
times: int = 2) -> None:
"""
Runs the google-java-format jar against all .java files found within the
provided path.
"""
jar_name = f"google-java-format-{version}.jar"
jar = cache.get_cache_dir() / jar_name
if not jar.exists():
_download_formatter(version, jar)
# Find all .java files in path and run the formatter on them
files = list(glob.iglob(os.path.join(path, "**/*.java"), recursive=True))
# Run the formatter as a jar file
logger.info("Running java formatter on {} files".format(len(files)))
for _ in range(times):
shell.run(["java", "-jar", str(jar), "--replace"] + files)
def owlbot_main(
template_path: Optional[Path] = None,
staging_excludes: Optional[List[str]] = None,
templates_excludes: Optional[List[str]] = None,
patch_staging: Callable[[Path], None] = _noop,
) -> None:
"""Copies files from staging and template directories into current working dir.
Args:
template_path: path to template directory; omit except in tests.
staging_excludes: paths to ignore when copying from the staging directory
templates_excludes: paths to ignore when copying generated templates
patch_staging: callback function runs on each staging directory before
copying it into repo root. Add your regular expression substitution code
here.
When there is no owlbot.py file, run this function instead. Also, when an
owlbot.py file is necessary, the first statement of owlbot.py should probably
call this function.
Depends on owl-bot copying into a staging directory, so your .Owlbot.yaml should
look a lot like this:
docker:
image: gcr.io/repo-automation-bots/owlbot-nodejs:latest
deep-remove-regex:
- /owl-bot-staging
deep-copy-regex:
- source: /google/cloud/video/transcoder/(.*)/.*-nodejs/(.*)
dest: /owl-bot-staging/$1/$2
Also, this function requires a default_version in your .repo-metadata.json. Ex:
"default_version": "v1",
"""
if staging_excludes is None:
staging_excludes = default_staging_excludes
if templates_excludes is None:
templates_excludes = default_templates_excludes
logging.basicConfig(level=logging.DEBUG)
# Load the default version defined in .repo-metadata.json.
default_version = json.load(open(".repo-metadata.json", "rt")).get(
"default_version"
)
staging = Path("owl-bot-staging")
s_copy = transforms.move
if default_version is None:
logger.info("No default version found in .repo-metadata.json. Ok.")
elif staging.is_dir():
logger.info(f"Copying files from staging directory ${staging}.")
# Collect the subdirectories of the staging directory.
versions = [v.name for v in staging.iterdir() if v.is_dir()]
# Reorder the versions so the default version always comes last.
versions = [v for v in versions if v != default_version] + [default_version]
logger.info(f"Collected versions ${versions} from ${staging}")
# Copy each version directory into the root.
for version in versions:
library = staging / version
_tracked_paths.add(library)
patch_staging(library)
s_copy([library], excludes=staging_excludes)
# The staging directory should never be merged into the main branch.
shutil.rmtree(staging)
else:
# Collect the subdirectories of the src directory.
src = Path("src")
versions = [v.name for v in src.iterdir() if v.is_dir()]
# Reorder the versions so the default version always comes last.
versions = [v for v in versions if v != default_version] + [default_version]
logger.info(f"Collected versions ${versions} from ${src}")
common_templates = gcp.CommonTemplates(template_path)
common_templates.excludes.extend(templates_excludes)
if default_version:
templates = common_templates.node_library(
source_location="build/src",
versions=versions,
default_version=default_version,
)
s_copy([templates], excludes=templates_excludes)
postprocess_gapic_library_hermetic()
else:
templates = common_templates.node_library(source_location="build/src")
s_copy([templates], excludes=templates_excludes)
def _include_samples(
self,
language: str,
version: str,
genfiles: Path,
googleapis_service_dir: Path,
samples_root_dir: Path = None,
samples_resources_dir: Path = None,
):
"""Include code samples and supporting resources in generated output.
Resulting directory structure in generated output:
samples/
├── resources
│ ├── example_text_file.txt
│ └── example_data.csv
└── v1/
├── sample_one.py
├── sample_two.py
└── test/
├── samples.manifest.yaml
├── sample_one.test.yaml
└── sample_two.test.yaml
Samples are included in the genfiles output of the generator.
Sample tests are defined in googleapis:
{service}/{version}/samples/test/*.test.yaml
Sample resources are declared in {service}/sample_resources.yaml
which includes a list of files with public gs:// URIs for download.
Sample resources are files needed to run code samples or system tests.
Synth keeps resources in sync by always pulling down the latest version.
It is recommended to store resources in the `cloud-samples-data` bucket.
Sample manifest is a generated file which defines invocation commands
for each code sample (used by sample-tester to invoke samples).
"""
if samples_root_dir is None:
samples_root_dir = genfiles / "samples"
if samples_resources_dir is None:
samples_resources_dir = samples_root_dir / "resources"
samples_version_dir = samples_root_dir / version
# Some languages capitalize their `V` prefix for version numbers
if not samples_version_dir.is_dir():
samples_version_dir = samples_root_dir / version.capitalize()
# Do not proceed if genfiles does not include samples/{version} dir.
if not samples_version_dir.is_dir():
return None
samples_test_dir = samples_version_dir / "test"
samples_manifest_yaml = samples_test_dir / "samples.manifest.yaml"
googleapis_samples_dir = googleapis_service_dir / version / "samples"
googleapis_resources_yaml = googleapis_service_dir / "sample_resources.yaml"
# Copy sample tests from googleapis {service}/{version}/samples/*.test.yaml
# into generated output as samples/{version}/test/*.test.yaml
test_files = googleapis_samples_dir.glob("**/*.test.yaml")
os.makedirs(samples_test_dir, exist_ok=True)
for i in test_files:
logger.debug(f"Copy: {i} to {samples_test_dir / i.name}")
shutil.copyfile(i, samples_test_dir / i.name)
# Download sample resources from sample_resources.yaml storage URIs.
#
# sample_resources:
# - uri: gs://bucket/the/file/path.csv
# description: Description of this resource
#
# Code follows happy path. An error is desirable if YAML is invalid.
if googleapis_resources_yaml.is_file():
with open(googleapis_resources_yaml, "r") as f:
resources_data = yaml.load(f, Loader=yaml.SafeLoader)
resource_list = resources_data.get("sample_resources")
for resource in resource_list:
uri = resource.get("uri")
if uri.startswith("gs://"):
uri = uri.replace("gs://", "https://storage.googleapis.com/")
response = requests.get(uri, allow_redirects=True)
download_path = samples_resources_dir / os.path.basename(uri)
os.makedirs(samples_resources_dir, exist_ok=True)
logger.debug(f"Download {uri} to {download_path}")
with open(download_path, "wb") as output: # type: ignore
output.write(response.content)
# Generate manifest file at samples/{version}/test/samples.manifest.yaml
# Includes a reference to every sample (via its "region tag" identifier)
# along with structured instructions on how to invoke that code sample.
relative_manifest_path = str(
samples_manifest_yaml.relative_to(samples_root_dir)
)
LANGUAGE_EXECUTABLES = {
"nodejs": "node",
"php": "php",
"python": "python3",
"ruby": "bundle exec ruby",
}
if language not in LANGUAGE_EXECUTABLES:
logger.info("skipping manifest gen")
return None
manifest_arguments = [
"gen-manifest",
f"--env={language}",
f"--bin={LANGUAGE_EXECUTABLES[language]}",
f"--output={relative_manifest_path}",
"--chdir={@manifest_dir}/../..",
]
for code_sample in samples_version_dir.glob("*"):
sample_path = str(code_sample.relative_to(samples_root_dir))
if os.path.isfile(code_sample):
manifest_arguments.append(sample_path)
try:
logger.debug(f"Writing samples manifest {manifest_arguments}")
shell.run(manifest_arguments, cwd=samples_root_dir)
except (subprocess.CalledProcessError, FileNotFoundError):
logger.warning("gen-manifest failed (sample-tester may not be installed)")
