def _remove_obsolete_files(old_metadata): """Remove obsolete files from the file system. Call add_new_files() before this function or it will remove all generated files. Parameters: old_metadata: old metadata loaded from a call to read_or_empty(). """ old_files = set(old_metadata.generated_files) new_files = set(_metadata.generated_files) excluded_patterns = set([pattern for pattern in _excluded_patterns]) obsolete_files = old_files - new_files for file_path in git_ignore(obsolete_files): try: matched_pattern = False for pattern in excluded_patterns: if fnmatch.fnmatch(file_path, pattern): matched_pattern = True break if matched_pattern: logger.info( f"Leaving obsolete file {file_path} because it matched excluded pattern {pattern} during copy." ) else: logger.info(f"Removing obsolete file {file_path}...") os.unlink(file_path) except FileNotFoundError: pass # Already deleted. That's OK.
def _download_formatter(version: str, dest: Path) -> None: logger.info("Downloading java formatter") url = JAR_DOWNLOAD_URL.format(version=version) response = requests.get(url) response.raise_for_status() with open(dest, "wb") as fh: fh.write(response.content)
def replace( sources: ListOfPathsOrStrs, before: str, after: str, flags: int = re.MULTILINE ) -> int: """Replaces occurrences of before with after in all the given sources. Returns: The number of times the text was found and replaced across all files. """ expr = re.compile(before, flags=flags or 0) paths = _filter_files(_expand_paths(sources, ".")) if not paths: logger.warning(f"No files were found in sources {sources} for replace()") count_replaced = 0 for path in paths: replaced = _replace_in_file(path, expr, after) count_replaced += replaced if replaced: logger.info(f"Replaced {before!r} in {path}.") if not count_replaced: logger.warning( f"No replacements made in {sources} for pattern {before}, maybe " "replacement is no longer needed?" ) return count_replaced
def owlbot_copy_version( src: Path, dest: Path, copy_excludes: typing.Optional[typing.List[str]] = None, ) -> None: """Copies files from a version subdirectory. """ logger.debug("owlbot_copy_version called from %s to %s", src, dest) if copy_excludes is None: copy_excludes = DEFAULT_COPY_EXCLUDES # detect the version string for later use src_dir = src / "src" entries = os.scandir(src_dir) if not entries: logger.info("there is no src directory '%s' to copy", src_dir) return version_string = os.path.basename(os.path.basename(next(entries))) logger.debug("version_string detected: %s", version_string) # copy all src including partial veneer classes s.move([src / "src"], dest / "src", merge=_merge, excludes=copy_excludes) # copy tests s.move([src / "tests"], dest / "tests", merge=_merge, excludes=copy_excludes) # detect the directory containing proto generated PHP source and metadata. proto_src = src / "proto/src" entries = os.scandir(proto_src) proto_dir = None metadata_dir = None if not entries: logger.info("there is no proto generated src directory to copy: %s", proto_src) return for entry in entries: if os.path.basename(entry.path) == METADATA_DIR: metadata_dir = _find_copy_target( Path(entry.path).resolve(), version_string) else: proto_dir = _find_copy_target( Path(entry.path).resolve(), version_string) # copy proto files if isinstance(proto_dir, Path): logger.debug("proto_dir detected: %s", proto_dir) s.move([proto_dir], dest / "src", merge=_merge, excludes=copy_excludes) # copy metadata files if isinstance(metadata_dir, Path): logger.debug("metadata_dir detected: %s", metadata_dir) s.move([metadata_dir], dest / "metadata", merge=_merge, excludes=copy_excludes)
def format_code(path: str, times: int = 2) -> None: """ Runs the google-java-format jar against all .java files found within the provided path. """ # Find all .java files in path and run the formatter on them files = list(glob.iglob(os.path.join(path, "**/*.java"), recursive=True)) # Run the formatter as a jar file logger.info("Running java formatter on {} files".format(len(files))) formatter_binary = sys.argv[2] for _ in range(times): shell.run([formatter_binary, "--replace"] + files)
def generate_index_ts(versions: List[str], default_version: str) -> None: """ generate src/index.ts to export the client name and versions in the client library. Args: versions: the list of versions, like: ['v1', 'v1beta1', ...] default_version: a stable version provided by API producer. It must exist in argument versions. Return: True/False: return true if successfully generate src/index.ts, vice versa. """ # sanitizer the input arguments if len(versions) < 1: err_msg = ( "List of version can't be empty, it must contain default version at least." ) logger.error(err_msg) raise AttributeError(err_msg) if default_version not in versions: err_msg = f"Version {versions} must contain default version {default_version}." logger.error(err_msg) raise AttributeError(err_msg) # To make sure the output is always deterministic. versions = sorted(versions) # compose default version's index.ts file path versioned_index_ts_path = Path("src") / default_version / "index.ts" clients = extract_clients(versioned_index_ts_path) if not clients: err_msg = f"No client is exported in the default version's({default_version}) index.ts ." logger.error(err_msg) raise AttributeError(err_msg) # compose template directory template_path = ( Path(__file__).parent.parent / "gcp" / "templates" / "node_split_library" ) template_loader = FileSystemLoader(searchpath=str(template_path)) template_env = Environment(loader=template_loader, keep_trailing_newline=True) TEMPLATE_FILE = "index.ts.j2" index_template = template_env.get_template(TEMPLATE_FILE) # render index.ts content output_text = index_template.render( versions=versions, default_version=default_version, clients=clients ) with open("src/index.ts", "w") as fh: fh.write(output_text) logger.info("successfully generate `src/index.ts`")
def owlbot_main( src: Path, dest: Path, copy_excludes: typing.Optional[typing.List[str]] = None, patch_func: typing.Callable[[], None] = owlbot_patch, ) -> None: """Copies files from generated tree. """ entries = os.scandir(src) if not entries: logger.info("there is no version subdirectory to copy") return for entry in entries: if entry.is_dir(): version_src = Path(entry.path).resolve() owlbot_copy_version(version_src, dest, copy_excludes) with pushd(dest): patch_func()
def _remove_obsolete_files(old_metadata): """Remove obsolete files from the file system. Call add_new_files() before this function or it will remove all generated files. Parameters: old_metadata: old metadata loaded from a call to read_or_empty(). """ old_files = set(old_metadata.generated_files) new_files = set(_metadata.generated_files) obsolete_files = old_files - new_files for file_path in git_ignore(obsolete_files): try: logger.info(f"Removing obsolete file {file_path}...") os.unlink(file_path) except FileNotFoundError: pass # Already deleted. That's OK.
def format_code(path: str, version: str = DEFAULT_FORMAT_VERSION, times: int = 2) -> None: """ Runs the google-java-format jar against all .java files found within the provided path. """ jar_name = f"google-java-format-{version}.jar" jar = cache.get_cache_dir() / jar_name if not jar.exists(): _download_formatter(version, jar) # Find all .java files in path and run the formatter on them files = list(glob.iglob(os.path.join(path, "**/*.java"), recursive=True)) # Run the formatter as a jar file logger.info("Running java formatter on {} files".format(len(files))) for _ in range(times): shell.run(["java", "-jar", str(jar), "--replace"] + files)
def owlbot_main( template_path: Optional[Path] = None, staging_excludes: Optional[List[str]] = None, templates_excludes: Optional[List[str]] = None, patch_staging: Callable[[Path], None] = _noop, ) -> None: """Copies files from staging and template directories into current working dir. Args: template_path: path to template directory; omit except in tests. staging_excludes: paths to ignore when copying from the staging directory templates_excludes: paths to ignore when copying generated templates patch_staging: callback function runs on each staging directory before copying it into repo root. Add your regular expression substitution code here. When there is no owlbot.py file, run this function instead. Also, when an owlbot.py file is necessary, the first statement of owlbot.py should probably call this function. Depends on owl-bot copying into a staging directory, so your .Owlbot.yaml should look a lot like this: docker: image: gcr.io/repo-automation-bots/owlbot-nodejs:latest deep-remove-regex: - /owl-bot-staging deep-copy-regex: - source: /google/cloud/video/transcoder/(.*)/.*-nodejs/(.*) dest: /owl-bot-staging/$1/$2 Also, this function requires a default_version in your .repo-metadata.json. Ex: "default_version": "v1", """ if staging_excludes is None: staging_excludes = default_staging_excludes if templates_excludes is None: templates_excludes = default_templates_excludes logging.basicConfig(level=logging.DEBUG) # Load the default version defined in .repo-metadata.json. default_version = json.load(open(".repo-metadata.json", "rt")).get( "default_version" ) staging = Path("owl-bot-staging") s_copy = transforms.move if default_version is None: logger.info("No default version found in .repo-metadata.json. Ok.") elif staging.is_dir(): logger.info(f"Copying files from staging directory ${staging}.") # Collect the subdirectories of the staging directory. versions = [v.name for v in staging.iterdir() if v.is_dir()] # Reorder the versions so the default version always comes last. versions = [v for v in versions if v != default_version] + [default_version] logger.info(f"Collected versions ${versions} from ${staging}") # Copy each version directory into the root. for version in versions: library = staging / version _tracked_paths.add(library) patch_staging(library) s_copy([library], excludes=staging_excludes) # The staging directory should never be merged into the main branch. shutil.rmtree(staging) else: # Collect the subdirectories of the src directory. src = Path("src") versions = [v.name for v in src.iterdir() if v.is_dir()] # Reorder the versions so the default version always comes last. versions = [v for v in versions if v != default_version] + [default_version] logger.info(f"Collected versions ${versions} from ${src}") common_templates = gcp.CommonTemplates(template_path) common_templates.excludes.extend(templates_excludes) if default_version: templates = common_templates.node_library( source_location="build/src", versions=versions, default_version=default_version, ) s_copy([templates], excludes=templates_excludes) postprocess_gapic_library_hermetic() else: templates = common_templates.node_library(source_location="build/src") s_copy([templates], excludes=templates_excludes)
def _include_samples( self, language: str, version: str, genfiles: Path, googleapis_service_dir: Path, samples_root_dir: Path = None, samples_resources_dir: Path = None, ): """Include code samples and supporting resources in generated output. Resulting directory structure in generated output: samples/ ├── resources │ ├── example_text_file.txt │ └── example_data.csv └── v1/ ├── sample_one.py ├── sample_two.py └── test/ ├── samples.manifest.yaml ├── sample_one.test.yaml └── sample_two.test.yaml Samples are included in the genfiles output of the generator. Sample tests are defined in googleapis: {service}/{version}/samples/test/*.test.yaml Sample resources are declared in {service}/sample_resources.yaml which includes a list of files with public gs:// URIs for download. Sample resources are files needed to run code samples or system tests. Synth keeps resources in sync by always pulling down the latest version. It is recommended to store resources in the `cloud-samples-data` bucket. Sample manifest is a generated file which defines invocation commands for each code sample (used by sample-tester to invoke samples). """ if samples_root_dir is None: samples_root_dir = genfiles / "samples" if samples_resources_dir is None: samples_resources_dir = samples_root_dir / "resources" samples_version_dir = samples_root_dir / version # Some languages capitalize their `V` prefix for version numbers if not samples_version_dir.is_dir(): samples_version_dir = samples_root_dir / version.capitalize() # Do not proceed if genfiles does not include samples/{version} dir. if not samples_version_dir.is_dir(): return None samples_test_dir = samples_version_dir / "test" samples_manifest_yaml = samples_test_dir / "samples.manifest.yaml" googleapis_samples_dir = googleapis_service_dir / version / "samples" googleapis_resources_yaml = googleapis_service_dir / "sample_resources.yaml" # Copy sample tests from googleapis {service}/{version}/samples/*.test.yaml # into generated output as samples/{version}/test/*.test.yaml test_files = googleapis_samples_dir.glob("**/*.test.yaml") os.makedirs(samples_test_dir, exist_ok=True) for i in test_files: logger.debug(f"Copy: {i} to {samples_test_dir / i.name}") shutil.copyfile(i, samples_test_dir / i.name) # Download sample resources from sample_resources.yaml storage URIs. # # sample_resources: # - uri: gs://bucket/the/file/path.csv # description: Description of this resource # # Code follows happy path. An error is desirable if YAML is invalid. if googleapis_resources_yaml.is_file(): with open(googleapis_resources_yaml, "r") as f: resources_data = yaml.load(f, Loader=yaml.SafeLoader) resource_list = resources_data.get("sample_resources") for resource in resource_list: uri = resource.get("uri") if uri.startswith("gs://"): uri = uri.replace("gs://", "https://storage.googleapis.com/") response = requests.get(uri, allow_redirects=True) download_path = samples_resources_dir / os.path.basename(uri) os.makedirs(samples_resources_dir, exist_ok=True) logger.debug(f"Download {uri} to {download_path}") with open(download_path, "wb") as output: # type: ignore output.write(response.content) # Generate manifest file at samples/{version}/test/samples.manifest.yaml # Includes a reference to every sample (via its "region tag" identifier) # along with structured instructions on how to invoke that code sample. relative_manifest_path = str( samples_manifest_yaml.relative_to(samples_root_dir) ) LANGUAGE_EXECUTABLES = { "nodejs": "node", "php": "php", "python": "python3", "ruby": "bundle exec ruby", } if language not in LANGUAGE_EXECUTABLES: logger.info("skipping manifest gen") return None manifest_arguments = [ "gen-manifest", f"--env={language}", f"--bin={LANGUAGE_EXECUTABLES[language]}", f"--output={relative_manifest_path}", "--chdir={@manifest_dir}/../..", ] for code_sample in samples_version_dir.glob("*"): sample_path = str(code_sample.relative_to(samples_root_dir)) if os.path.isfile(code_sample): manifest_arguments.append(sample_path) try: logger.debug(f"Writing samples manifest {manifest_arguments}") shell.run(manifest_arguments, cwd=samples_root_dir) except (subprocess.CalledProcessError, FileNotFoundError): logger.warning("gen-manifest failed (sample-tester may not be installed)")