def build_index(index_type: str, tf_dir: str, output_file: str, overview_desc: str): """ Update the documentation index. Parameters: ---------- index_type (str): Either 'Components' or 'Catalog'. tf_dir (str): The directory to scan for modules. output_file (str): The location of the documentation file to create. overview_dsc (str): The overview that will appear at the top of the index. """ content_metadata = { "aws": "", # "azure": "", # "gcp": "", } git_url_pattern = "git::{git_repo}/{path}?ref={branch}" git_repo = "https://github.com/slalom-ggp/dataops-infra" git_branch = "main" toc_str = "" for platform_i, platform in enumerate(content_metadata.keys(), start=1): toc_str += (f"{platform_i}. [{_proper(platform)} {index_type}]" f"(#{platform.lower()}-{index_type.lower()})\n") logging.info(f"Exploring platform '{platform}'") catalog_modules = get_tf_metadata(f"{tf_dir}/{platform}", recursive=True) for module, metadata in catalog_modules.items(): module_title = f"{_proper(platform)} {_proper(os.path.basename(module))}" toc_str += ( f" - [{module_title}](#{module_title.replace(' ', '-').lower()})\n" ) logging.debug(f"Exploring module '{module}': {metadata}") readme_path = f"{module}/README.md" content_metadata[platform] += ( f"### {module_title}\n\n" f"#### Overview\n\n" f"{metadata['header']}\n\n" f"#### Documentation\n\n" f"- [{module_title} Readme]({readme_path})\n\n" f"-------------------\n\n") content = CATALOG_TEMPLATE.format( toc=toc_str, aws=content_metadata["aws"], azure="_(Coming soon)_", gcp="_(Coming soon)_", index_type=index_type, overview=overview_desc, ) uio.create_text_file(output_file, content)
def change_upstream_source( dir_to_update=".", git_repo="https://github.com/slalom-ggp/dataops-infra", branch="master", relative_path="../../dataops-infra", to_relative=False, to_git=False, dry_run=False, ): """Change Terraform source""" if to_relative and to_git or not (to_relative or to_git): raise ValueError( "Must specify `--to_git` or `--to_relative`, but not both.") for tf_file in uio.list_local_files(dir_to_update, recursive=False): if tf_file.endswith(".tf"): # print(tf_file) new_lines = [] for line in uio.get_text_file_contents(tf_file).splitlines(): new_line = line if line.lstrip().startswith("source "): current_path = line.lstrip().split('"')[1] start_pos = max([ current_path.find("catalog/"), current_path.find("components/") ]) if start_pos > 0: module_path = current_path[start_pos:].split( "?ref=")[0] if to_relative: local_patten = "{relative_path}/{path}" new_path = local_patten.format( relative_path=relative_path, path=module_path) elif to_git: git_pattern = "git::{git_repo}//{path}?ref={branch}" new_path = git_pattern.format(git_repo=git_repo, path=module_path, branch=branch) if current_path == new_path: print(f"{current_path} \n\t\t\t-> (unchanged)") else: print(f"{current_path} \n\t\t\t-> {new_path}") new_line = f' source = "{new_path}"' new_lines.append(new_line) new_file_text = "\n".join(new_lines) if dry_run: print(f"\n\n------------\n-- {tf_file}\n------------") print(new_file_text) else: uio.create_text_file(tf_file, new_file_text) if not dry_run: runnow.run("terraform fmt -recursive", dir_to_update)
def _discover( tap_name: str, taps_dir: str, *, config_file: str, catalog_dir: str, dockerized: bool, tap_exe: str, ) -> None: catalog_file = config.get_raw_catalog_file( taps_dir, catalog_dir, tap_name, allow_custom=False ) uio.create_folder(catalog_dir) img = f"{docker.BASE_DOCKER_REPO}:{tap_exe}" hide_cmd = False if dockerized: cdw = os.getcwd().replace("\\", "/") tap_config = json.loads(uio.get_text_file_contents(config_file)) tap_docker_args = "" # TODO: Replace with logic to parse from AWS_SHARED_CREDENTIALS_FILE env var: for k in ["aws_access_key_id", "aws_secret_access_key", "aws_session_token"]: if k in tap_config: key = f"TAP_{tap_name}_{k}".replace("-", "_").upper() os.environ[key] = tap_config[k] tap_docker_args += f' -e {k.upper()}="{tap_config[k]}"' hide_cmd = True _, _ = runnow.run(f"docker pull {img}") _, output_text = runnow.run( f"docker run --rm -i " f"-v {cdw}:/home/local {tap_docker_args} " f"{img} --config {config.dockerize_cli_args(config_file)} --discover", echo=False, capture_stderr=False, hide=hide_cmd, ) if not _is_valid_json(output_text): raise RuntimeError(f"Could not parse json file from output:\n{output_text}") uio.create_text_file(catalog_file, output_text) else: runnow.run( f"{tap_exe} --config {config_file} --discover > {catalog_file}", hide=hide_cmd, )
def smart_split(dockerfile_path: str, tag_as, addl_args=None): tag_as = _to_list(tag_as) if tag_as: interim_image_name = tag_as[0].split(":")[0] else: interim_image_name = "untitled_image" (image_core, dockerfile_core), (image_derived, dockerfile_derived) = _smart_split(dockerfile_path, interim_image_name, addl_args=addl_args) dockerfile_path_core = os.path.realpath(f"{dockerfile_path}.core") dockerfile_path_derived = os.path.realpath(f"{dockerfile_path}.quick") uio.create_text_file(filepath=dockerfile_path_core, contents=dockerfile_core) if dockerfile_derived: uio.create_text_file(filepath=dockerfile_path_derived, contents=dockerfile_derived) else: uio.delete_file(dockerfile_path_derived, ignore_missing=True) dockerfile_path_derived = None return image_core, dockerfile_path_core, image_derived, dockerfile_path_derived
def make_aggregate_state_file(raw_json_lines_file: str, output_json_file: str) -> None: """ Create a valid json state file from one or more json lines ('jsonl' format). Parameters ---------- raw_json_lines_file : str Path to a jsonl (json lines) file containing one or more json documents to aggregate. output_json_file : str Path to use when saving the aggregated json file. """ try: uio.create_text_file( output_json_file, get_aggregate_state( uio.get_text_file_contents(raw_json_lines_file)), ) except ValueError as ex: raise ValueError( f"State file from '{raw_json_lines_file}' is not valid JSON or JSONL. " f"Please either delete or fix the file and then retry. {ex}")
def get_tf_metadata(tf_dir: str, recursive: bool = False, save_to_dir: bool = True): """ Return a dictionary of Terraform module paths to JSON metadata about each module, a wrapper around the `terraform-docs` tool. Parameters: ---------- tf_dir: Directory of terraform scripts to scan. recursive : Optional (default=True). 'True' to run on all subdirectories, recursively. Returns: ------- dict """ result = {} if (".git" not in tf_dir and ".terraform" not in tf_dir and "samples" not in tf_dir and "tests" not in tf_dir): if [ x for x in uio.list_local_files(tf_dir, recursive=False) if x.endswith(".tf") ]: _, json_text = runnow.run(f"terraform-docs json {tf_dir}", echo=False) result[tf_dir] = json.loads(json_text) if save_to_dir: uio.create_folder(f"{tf_dir}/.terraform") uio.create_text_file( f"{tf_dir}/.terraform/terraform-docs.json", json_text) if recursive: for folder in uio.list_local_files(tf_dir, recursive=False): folder = folder.replace("\\", "/") if os.path.isdir(folder): result.update(get_tf_metadata(folder, recursive=recursive)) return result
def _update_var_output(output_var): return_code, output = runnow.run(f"terraform output {output_var}", echo=False) uio.create_text_file(os.path.join("outputs", output_var), contents=output) return True
def update_module_docs( tf_dir: str, *, recursive: bool = True, readme: str = "README.md", footer: bool = True, header: bool = True, special_case_words: List[str] = None, extra_docs_names: List[str] = ["USAGE.md", "NOTES.md"], git_repo: str = "https://github.com/slalom-ggp/dataops-infra", ): """ Replace all README.md files with auto-generated documentation, a wrapper around the `terraform-docs` tool. Parameters: ---------- tf_dir: Directory of terraform scripts to document. recursive : Optional (default=True). 'True' to run on all subdirectories, recursively. readme : Optional (default="README.md"). The filename to create when generating docs. footnote: Optional (default=True). 'True' to include the standard footnote. special_case_words: Optional. A list of words to override special casing rules. extra_docs_names: (Optional.) A list of filenames which, if found, will be appended to each module's README.md file. git_repo: Optional. The git repo path to use in rendering 'source' paths. Returns: ------- None """ markdown_text = "" if ".git" not in tf_dir and ".terraform" not in tf_dir: tf_files = [ x for x in uio.list_local_files(tf_dir, recursive=False) if x.endswith(".tf") ] extra_docs = [ x for x in uio.list_local_files(tf_dir, recursive=False) if extra_docs_names and os.path.basename(x) in extra_docs_names ] if tf_files: module_title = _proper(os.path.basename(tf_dir), special_case_words=special_case_words) parent_dir_name = os.path.basename(Path(tf_dir).parent) if parent_dir_name != ".": module_title = _proper( f"{parent_dir_name} {module_title}", special_case_words=special_case_words, ) module_path = tf_dir.replace(".", "").replace("//", "/").replace("\\", "/") _, markdown_output = runnow.run( f"terraform-docs md --no-providers --sort-by-required {tf_dir}", echo=False, ) if header: markdown_text += DOCS_HEADER.format(module_title=module_title, module_path=module_path) markdown_text += markdown_output for extra_file in extra_docs: markdown_text += uio.get_text_file_contents(extra_file) + "\n" if footer: markdown_text += DOCS_FOOTER.format(src="\n".join([ "* [{file}]({repo}/tree/master/{dir}/{file})".format( repo=git_repo, dir=module_path, file=os.path.basename(tf_file), ) for tf_file in tf_files ])) uio.create_text_file(f"{tf_dir}/{readme}", markdown_text) if recursive: for folder in uio.list_local_files(tf_dir, recursive=False): if os.path.isdir(folder): update_module_docs(folder, recursive=recursive, readme=readme)
def update_module_docs( tf_dir: str, *, recursive: bool = True, readme: str = "README.md", footer: bool = True, header: bool = True, special_case_words: List[str] = None, extra_docs_names: List[str] = ["USAGE.md", "NOTES.md"], git_repo: str = "https://github.com/slalom-ggp/dataops-infra", ) -> None: """ Replace all README.md files with auto-generated documentation This is a wrapper around the `terraform-docs` tool. Parameters ---------- tf_dir : str Directory of terraform scripts to document. recursive : bool, optional Run on all subdirectories, recursively. By default True. readme : str, optional The filename to create when generating docs, by default "README.md". footer : bool, optional Include the standard footnote, by default True. header : bool, optional Include the standard footnote, by default True. special_case_words : List[str], optional A list of words to override special casing rules, by default None. extra_docs_names : List[str], optional A list of filenames which, if found, will be appended to each module's README.md file, by default ["USAGE.md", "NOTES.md"]. git_repo : str, optional The git repo path to use in rendering 'source' paths, by default "https://github.com/slalom-ggp/dataops-infra". """ markdown_text = "" if ".git" not in tf_dir and ".terraform" not in tf_dir: tf_files = [ x for x in uio.list_local_files(tf_dir, recursive=False) if x.endswith(".tf") ] extra_docs = [ x for x in uio.list_local_files(tf_dir, recursive=False) if extra_docs_names and os.path.basename(x) in extra_docs_names ] if tf_files: module_title = _proper(os.path.basename(tf_dir), special_case_words=special_case_words) parent_dir_name = os.path.basename(Path(tf_dir).parent) # parent_title = _proper( # parent_dir_name, special_case_words=special_case_words, # ) module_title = _proper( f"{parent_dir_name} {module_title}", special_case_words=special_case_words, ) module_path = tf_dir.replace(".", "").replace("//", "/").replace("\\", "/") _, markdown_output = runnow.run( f"terraform-docs markdown document --sort=false {tf_dir}", # " --no-requirements" echo=False, ) if "components" in module_path.lower(): module_type = "Components" elif "catalog" in module_path.lower(): module_type = "Catalog" else: module_type = "Other" if header: markdown_text += DOCS_HEADER.format( module_title=module_title, module_path=module_path, module_type=module_type, ) markdown_text += markdown_output for extra_file in extra_docs: markdown_text += uio.get_text_file_contents(extra_file) + "\n" if footer: markdown_text += DOCS_FOOTER.format(src="\n".join([ "* [{file}]({repo}/tree/main/{dir}/{file})".format( repo=git_repo, dir=module_path, file=os.path.basename(tf_file), ) for tf_file in tf_files ])) uio.create_text_file(f"{tf_dir}/{readme}", markdown_text) if recursive: for folder in uio.list_local_files(tf_dir, recursive=False): if os.path.isdir(folder): update_module_docs(folder, recursive=recursive, readme=readme)
def _infer_schema( tap_name: str, taps_dir: str, raw_catalog_file: str, selected_catalog_file: str, *, config_file: str, catalog_dir: str, dockerized: bool, tap_exe: str, ) -> str: custom_catalog = json.loads(uio.get_text_file_contents(raw_catalog_file)) tmp_folder = f"{catalog_dir}/tmp" tmp_outfile = f"{catalog_dir}/tmp/sync-dryrun.jsonl" uio.create_folder(catalog_dir) uio.create_folder(tmp_folder) logging.info(f"Cleaning up old files in tmp folder '{tmp_folder}'...") for file in uio.list_files(tmp_folder): if any( [ file.endswith(x) for x in ["-config.json", "-dryrun.jsonl", "-table.inferred.json"] ] ): uio.delete_file(file) img = f"{docker.BASE_DOCKER_REPO}:{tap_exe}" hide_cmd = False if dockerized: cdw = os.getcwd().replace("\\", "/") tap_config = json.loads(uio.get_text_file_contents(config_file)) tap_docker_args = "" # TODO: Replace with logic to parse from AWS_SHARED_CREDENTIALS_FILE env var: for k in ["aws_access_key_id", "aws_secret_access_key", "aws_session_token"]: if k in tap_config: key = f"TAP_{tap_name}_{k}".replace("-", "_").upper() os.environ[key] = tap_config[k] tap_docker_args += f' -e {k.upper()}="{tap_config[k]}"' hide_cmd = True _, _ = runnow.run(f"docker pull {img}") _, jsonl_out = runnow.run( f"docker run --rm -i " f"-v {cdw}:/home/local {tap_docker_args} " f"{img} " f"--config {config.dockerize_cli_args(config_file)}" f"--catalog {selected_catalog_file}", hide=hide_cmd, echo=False, capture_stderr=False, ) else: _, jsonl_out = runnow.run( f"{tap_exe} " f"--config {config_file} " f"--catalog {selected_catalog_file}", hide=hide_cmd, echo=False, capture_stderr=False, ) uio.create_text_file(tmp_outfile, jsonl_out) _, jsonl_out = runnow.run( f"cat {tmp_outfile} | singer-infer-schema --out-dir {tmp_folder}", ) for file in uio.list_files(tmp_folder): if file.endswith(".inferred.json"): logging.info(f"Parsing inferred schema from '{file}'...") inferred_schema = json.loads(uio.get_text_file_contents(file)) stream_name = file.split("/")[-1].split(".")[0] stream = ( [x for x in custom_catalog["streams"] if x["stream"] == stream_name] or [None] )[0] if not stream: raise ValueError( f"Failed to append inferred schema for stream name '{stream_name}'." f" Stream not present in catalog file {selected_catalog_file}." ) stream["schema"] = inferred_schema custom_catalog_file = config.get_custom_catalog_file(taps_dir, tap_name) uio.create_text_file(custom_catalog_file, json.dumps(custom_catalog, indent=2)) return custom_catalog_file
def _check_rules( tap_name: str, catalog_file: str, rules_file: str, replication_strategy: str, plan_file_out: str, selected_catalog_file_out: str, log_dir: Optional[str], ) -> None: """ Create plan file and selected catalog file from provided rules and raw catalog. Parameters ---------- catalog_file : str Path to a catalog file. rules_file : str Path to a rules file. plan_file_out : str Path to save the plan file. selected_catalog_file_out : str Path to save the selected catalog file. """ select_rules = [ line.split("#")[0].rstrip() for line in uio.get_text_file_contents(rules_file).splitlines() if line.split("#")[0].rstrip() ] matches: Dict[str, dict] = {} excluded_table_stream_ids: Dict[str, List[str]] = {} matched_stream_ids: Dict[str, str] = {} for stream_id, table_object in _get_catalog_tables_dict(catalog_file).items(): table_name = _get_stream_name(table_object) if _table_match_check( table_name=table_name, stream_id=stream_id, select_rules=select_rules, ): if table_name in matched_stream_ids: raise RuntimeError( f"Table name '{table_name}' matched multiple stream IDs: " f'"{matched_stream_ids[table_name]}" and "{stream_id}". ' "This is most often caused by tables with the same name under " "different source database schemas. Please qualify or disqualify " "specific stream name patterns by using double-quoted stream IDs " "in your rules file instead of or in addition to bare table names." ) matched_stream_ids[table_name] = stream_id matches[table_name] = {} for col_object in _get_catalog_table_columns(table_object): col_name = col_object matches[table_name][col_name] = _col_match_check( table_name, stream_id, col_name, select_rules ) else: if table_name in excluded_table_stream_ids: excluded_table_stream_ids[table_name].append(stream_id) else: excluded_table_stream_ids[table_name] = [stream_id] all_matches_lower = [m.lower() for m in matches.keys()] + [ f'"{m.lower()}"' for m in matched_stream_ids.values() ] declared_tables = set( [ rule.split(".")[0].rstrip().lstrip("!") for rule in select_rules if rule.split(".")[0].rstrip() and ("*" not in rule.split(".")[0]) ] ) for required_table in declared_tables: if required_table.lower() not in all_matches_lower: logging.warning( f"The table '{required_table}' was declared in the rules file " "but could not be found in the catalog." ) for match, match_cols in matches.items(): if not match_cols: logging.warning( f"The table '{match}' was declared in the rules file " "but did not match with any columns in the catalog." ) primary_keys, replication_keys = _get_table_keys( matches, matched_stream_ids, catalog_file, rules_file ) file_text = _make_plan_file_text( matches, primary_keys, replication_keys, matched_stream_ids, excluded_table_stream_ids, ) logging.info(f"Updating plan file: {plan_file_out}") uio.create_text_file(plan_file_out, file_text) config.push_logs(log_dir, [rules_file, plan_file_out]) _create_selected_catalog( tap_name, plan_file=plan_file_out, raw_catalog_file=catalog_file, output_file=selected_catalog_file_out, replication_strategy=replication_strategy, skip_senseless_validators=SKIP_SENSELESS_VALIDATORS, ) config.push_logs(log_dir, [selected_catalog_file_out])