def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) parser.add_argument( "--schema-file", required=True, type=str, help="""Location of schema.yml file. Where new source tables should be created. """, ) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 _, _, sources = check_refs_sources(paths=args.filenames, manifest=manifest) status_code = create_missing_sources(sources, output_path=args.schema_file) return status_code
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 status_code = 0 for filename in args.filenames: file = Path(filename) sql = file.read_text() status_code_file, tables = has_table_name(sql, filename) if status_code_file: status_code = status_code_file to_replace = itertools.chain( get_ref_from_name(manifest, tables), get_source_from_name(manifest, tables), get_unknown_source(tables), ) for replacement in to_replace: sql = re.sub(*replacement, sql, re.IGNORECASE) file.write_text(sql, encoding="utf-8") return status_code
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) args = parser.parse_args(argv) return has_loader(paths=args.filenames)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) parser.add_argument( "--tests", nargs="+", required=True, help="List of acceptable tests.", ) parser.add_argument( "--test-cnt", type=int, default=1, help="Minimum number of tests required.", ) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 return check_test_cnt( paths=args.filenames, manifest=manifest, test_group=args.tests, test_cnt=args.test_cnt, )
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) args = parser.parse_args(argv) return check_column_desc(paths=args.filenames)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) parser.add_argument( "--meta-keys", nargs="+", required=True, help="List of required key in meta part of source.", ) args = parser.parse_args(argv) return has_meta_key(paths=args.filenames, meta_keys=args.meta_keys)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) parser.add_argument( "--tags", nargs="+", required=True, help="A list of tags that source can have.", ) args = parser.parse_args(argv) return validate_tags(paths=args.filenames, tags=args.tags)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_catalog_args(parser) args = parser.parse_args(argv) try: catalog = get_json(args.catalog) except JsonOpenError as e: print(f"Unable to load catalog file ({e})") return 1 return check_source_columns(paths=args.filenames, catalog=catalog)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 return has_description(paths=args.filenames, manifest=manifest)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) args = parser.parse_args(argv) status_code = 0 for filename in args.filenames: with open(filename, "rb+") as file_obj: status_code_file = check_semicolon(file_obj, replace=True) if status_code_file: print(f"Replacing semicolon in {filename}.") status_code = status_code_file return status_code
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) group = parser.add_mutually_exclusive_group() group.add_argument( "--ignore", nargs="*", help= "Columns for which do not check whether have a different description.", ) args = parser.parse_args(argv) return replace_column_desc(paths=args.filenames, ignore=args.ignore)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 status_code, _, _ = check_refs_sources(paths=args.filenames, manifest=manifest) return status_code
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) parser.add_argument( "--freshness", nargs="+", required=True, choices=["warn_after", "error_after"], help="List of required freshness options.", ) args = parser.parse_args(argv) return has_freshness(paths=args.filenames, required_freshness=set(args.freshness))
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_dbt_cmd_args(parser) add_dbt_cmd_model_args(parser) args = parser.parse_args(argv) cmd = prepare_cmd( args.filenames, args.global_flags, args.cmd_flags, args.model_prefix, args.model_postfix, ) return run_dbt_cmd(cmd)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) args = parser.parse_args(argv) status_code = 0 for filename in args.filenames: # Read as binary so we can read byte-by-byte with open(filename, "rb+") as file_obj: status_code_file = check_semicolon(file_obj) if status_code_file: print(f"{filename}: contains a semicolon at the end. " f"dbt does not support that.") status_code = status_code_file return status_code
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) add_catalog_args(parser) parser.add_argument( "--properties-file", required=True, type=str, help="""Location of file where new model properties should be generated. Suffix has to be `yml` or `yaml`. It can also include {database}, {schema}, {name} and {alias} variables. E.g. /models/{schema}/{name}.yml for model `foo.bar` will create properties file in /models/foo/bar.yml. """, ) args = parser.parse_args(argv) if not Path(args.properties_file).suffix in [".yml", ".yaml"]: print( "Input parameter `--schema-file` has to" " contain `.yml` or `.yaml` extension." ) return 1 try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 try: catalog = get_json(args.catalog) except JsonOpenError as e: print(f"Unable to load catalog file ({e})") return 1 status_code = generate_properties_file( paths=args.filenames, manifest=manifest, catalog=catalog, properties_file=args.properties_file, ) return status_code
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) parser.add_argument( "--tests", metavar="KEY=VALUE", nargs="+", required=True, help="Set a number of key-value pairs." " Key is type of test (data or schema) and value is required " "minimal number of tests eg. --test data=1 schema=2" "(do not put spaces before or after the = sign). " "", action=ParseDict, ) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 required_tests = {} for test_type, cnt in args.tests.items(): allowed_types = ["data", "schema"] if test_type not in allowed_types: parser.error( f"Test type {test_type} is not one of allowed types {allowed_types}" ) try: test_cnt = int(cnt) except ValueError: parser.error(f"Unable to cast {cnt} to int.") required_tests[test_type] = test_cnt return check_test_cnt( paths=args.filenames, manifest=manifest, required_tests=required_tests )
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) args = parser.parse_args(argv) status_code = 0 for filename in args.filenames: sql = Path(filename).read_text() status_code_file, tables = has_table_name(sql, filename) if status_code_file: result = "\n- ".join(list(tables)) # pragma: no mutate print( f"{filename}: " f"does not use source() or ref() macros for tables:\n- {result}", ) status_code = status_code_file return status_code
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) parser.add_argument( "--min-child-cnt", type=int, default=0, help="Minimal number of child relations.", ) parser.add_argument( "--max-child-cnt", type=int, help="Miximal number of child relations.", ) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 required_cnt = [ { "operator": operator.lt, "type": "min", "dep": "childs", "cnt": args.min_child_cnt, }, { "operator": operator.gt, "type": "max", "dep": "childs", "cnt": args.max_child_cnt, }, ] return check_child_parent_cnt(paths=args.filenames, manifest=manifest, required_cnt=required_cnt)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) white_black = parser.add_mutually_exclusive_group() white_black.add_argument( "--whitelist", nargs="+", help="Whitelisted schemas.", ) white_black.add_argument( "--blacklist", nargs="+", help="Blacklisted schemas.", ) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 if not (args.blacklist or args.whitelist): print( "Please specify at least one `--blacklist` or `--whitelist` option." ) return 1 return check_parents_schema( paths=args.filenames, manifest=manifest, blacklist=args.blacklist, whitelist=args.whitelist, )
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) parser.add_argument( "--tags", nargs="+", required=True, help="A list of tags that models can have.", ) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 return validate_tags(paths=args.filenames, manifest=manifest, tags=args.tags)
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() add_filenames_args(parser) add_manifest_args(parser) parser.add_argument( "--meta-keys", nargs="+", required=True, help="List of required key in meta part of model.", ) args = parser.parse_args(argv) try: manifest = get_json(args.manifest) except JsonOpenError as e: print(f"Unable to load manifest file ({e})") return 1 return has_meta_key(paths=args.filenames, manifest=manifest, meta_keys=args.meta_keys)