def replace(sources: ListOfPathsOrStrs,
before: str,
after: str,
flags: int = re.MULTILINE) -> int:
"""Replaces occurrences of before with after in all the given sources.
Returns:
The number of times the text was found and replaced across all files.
"""
expr = re.compile(before, flags=flags or 0)
paths = _filter_files(_expand_paths(sources, "."))
if not paths:
log.warning(f"No files were found in sources {sources} for replace()")
count_replaced = 0
for path in paths:
replaced = _replace_in_file(path, expr, after)
count_replaced += replaced
if replaced:
log.info(f"Replaced {before!r} in {path}.")
if not count_replaced:
log.warning(
f"No replacements made in {sources} for pattern {before}, maybe "
"replacement is not longer needed?")
return count_replaced
def move(
sources: ListOfPathsOrStrs,
destination: PathOrStr = None,
excludes: ListOfPathsOrStrs = None,
merge: Callable[[str, str, Path], str] = None,
) -> bool:
"""
copy file(s) at source to current directory.
Returns: True if any files were copied, False otherwise.
"""
copied = False
for source in _expand_paths(sources):
if destination is None:
canonical_destination = _tracked_paths.relativize(source)
else:
canonical_destination = Path(destination)
if excludes:
excludes = [
_tracked_paths.relativize(e) for e in _expand_paths(excludes, source)
]
else:
excludes = []
if source.is_dir():
copied = copied or _copy_dir_to_existing_dir(
source, canonical_destination, excludes=excludes, merge=merge
)
elif source not in excludes:
# copy individual file
if merge is not None and canonical_destination.is_file():
_merge_file(source, canonical_destination, merge)
else:
shutil.copy2(source, canonical_destination)
copied = True
if not copied:
log.warning(
f"No files in sources {sources} were copied. Does the source "
f"contain files?"
)
return copied
def _get_new_files(newer_than: float) -> List[str]:
"""Searchs current directory for new files and returns them in a list.
Parameters:
newer_than: any file modified after this timestamp (from time.time())
will be added to the metadata
"""
new_files = []
for (root, dirs, files) in os.walk(os.getcwd()):
for filename in files:
filepath = os.path.join(root, filename)
try:
mtime = os.path.getmtime(filepath)
except FileNotFoundError:
log.warning(
f"FileNotFoundError while getting modified time for {filepath}."
)
continue
if mtime >= newer_than:
new_files.append(os.path.relpath(filepath))
return new_files
def replace(sources: ListOfPathsOrStrs,
before: str,
after: str,
flags: int = re.MULTILINE):
"""Replaces occurrences of before with after in all the given sources."""
expr = re.compile(before, flags=flags or 0)
paths = _filter_files(_expand_paths(sources, "."))
if not paths:
log.warning(f"No files were found in sources {sources} for replace()")
any_replaced = False
for path in paths:
replaced = _replace_in_file(path, expr, after)
any_replaced = any_replaced or replaced
if replaced:
log.info(f"Replaced {before!r} in {path}.")
if not any_replaced:
log.warning(
f"No replacements made in {sources} for pattern {before}, maybe "
"replacement is not longer needed?")
def _include_samples(
self,
language: str,
version: str,
genfiles: Path,
googleapis_service_dir: Path,
samples_root_dir: Path = None,
samples_resources_dir: Path = None,
):
"""Include code samples and supporting resources in generated output.
Resulting directory structure in generated output:
samples/
├── resources
│ ├── example_text_file.txt
│ └── example_data.csv
└── v1/
├── sample_one.py
├── sample_two.py
└── test/
├── samples.manifest.yaml
├── sample_one.test.yaml
└── sample_two.test.yaml
Samples are included in the genfiles output of the generator.
Sample tests are defined in googleapis:
{service}/{version}/samples/test/*.test.yaml
Sample resources are declared in {service}/sample_resources.yaml
which includes a list of files with public gs:// URIs for download.
Sample resources are files needed to run code samples or system tests.
Synth keeps resources in sync by always pulling down the latest version.
It is recommended to store resources in the `cloud-samples-data` bucket.
Sample manifest is a generated file which defines invocation commands
for each code sample (used by sample-tester to invoke samples).
"""
if samples_root_dir is None:
samples_root_dir = genfiles / "samples"
if samples_resources_dir is None:
samples_resources_dir = samples_root_dir / "resources"
samples_version_dir = samples_root_dir / version
# Some languages capitalize their `V` prefix for version numbers
if not samples_version_dir.is_dir():
samples_version_dir = samples_root_dir / version.capitalize()
# Do not proceed if genfiles does not include samples/{version} dir.
if not samples_version_dir.is_dir():
return None
samples_test_dir = samples_version_dir / "test"
samples_manifest_yaml = samples_test_dir / "samples.manifest.yaml"
googleapis_samples_dir = googleapis_service_dir / version / "samples"
googleapis_resources_yaml = googleapis_service_dir / "sample_resources.yaml"
# Copy sample tests from googleapis {service}/{version}/samples/*.test.yaml
# into generated output as samples/{version}/test/*.test.yaml
test_files = googleapis_samples_dir.glob("**/*.test.yaml")
os.makedirs(samples_test_dir, exist_ok=True)
for i in test_files:
log.debug(f"Copy: {i} to {samples_test_dir / i.name}")
shutil.copyfile(i, samples_test_dir / i.name)
# Download sample resources from sample_resources.yaml storage URIs.
#
# sample_resources:
# - uri: gs://bucket/the/file/path.csv
# description: Description of this resource
#
# Code follows happy path. An error is desirable if YAML is invalid.
if googleapis_resources_yaml.is_file():
with open(googleapis_resources_yaml, "r") as f:
resources_data = yaml.load(f, Loader=yaml.SafeLoader)
resource_list = resources_data.get("sample_resources")
for resource in resource_list:
uri = resource.get("uri")
if uri.startswith("gs://"):
uri = uri.replace("gs://",
"https://storage.googleapis.com/")
response = requests.get(uri, allow_redirects=True)
download_path = samples_resources_dir / os.path.basename(uri)
os.makedirs(samples_resources_dir, exist_ok=True)
log.debug(f"Download {uri} to {download_path}")
with open(download_path, "wb") as output: # type: ignore
output.write(response.content)
# Generate manifest file at samples/{version}/test/samples.manifest.yaml
# Includes a reference to every sample (via its "region tag" identifier)
# along with structured instructions on how to invoke that code sample.
relative_manifest_path = str(
samples_manifest_yaml.relative_to(samples_root_dir))
LANGUAGE_EXECUTABLES = {
"nodejs": "node",
"php": "php",
"python": "python3",
"ruby": "bundle exec ruby",
}
if language not in LANGUAGE_EXECUTABLES:
log.info("skipping manifest gen")
return None
manifest_arguments = [
"gen-manifest",
f"--env={language}",
f"--bin={LANGUAGE_EXECUTABLES[language]}",
f"--output={relative_manifest_path}",
"--chdir={@manifest_dir}/../..",
]
for code_sample in samples_version_dir.glob("*"):
sample_path = str(code_sample.relative_to(samples_root_dir))
if os.path.isfile(code_sample):
manifest_arguments.append(sample_path)
try:
log.debug(f"Writing samples manifest {manifest_arguments}")
shell.run(manifest_arguments, cwd=samples_root_dir)
except (subprocess.CalledProcessError, FileNotFoundError):
log.warning(
"gen-manifest failed (sample-tester may not be installed)")
