def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--schema-file",
required=True,
type=str,
help="""Location of schema.yml file. Where new source tables should
be created.
""",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
_, _, sources = check_refs_sources(paths=args.filenames, manifest=manifest)
status_code = create_missing_sources(sources, output_path=args.schema_file)
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
status_code = 0
for filename in args.filenames:
file = Path(filename)
sql = file.read_text()
status_code_file, tables = has_table_name(sql, filename)
if status_code_file:
status_code = status_code_file
to_replace = itertools.chain(
get_ref_from_name(manifest, tables),
get_source_from_name(manifest, tables),
get_unknown_source(tables),
)
for replacement in to_replace:
sql = re.sub(*replacement, sql, re.IGNORECASE)
file.write_text(sql, encoding="utf-8")
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
args = parser.parse_args(argv)
return has_loader(paths=args.filenames)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--tests",
nargs="+",
required=True,
help="List of acceptable tests.",
)
parser.add_argument(
"--test-cnt",
type=int,
default=1,
help="Minimum number of tests required.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return check_test_cnt(
paths=args.filenames,
manifest=manifest,
test_group=args.tests,
test_cnt=args.test_cnt,
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
args = parser.parse_args(argv)
return check_column_desc(paths=args.filenames)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
parser.add_argument(
"--meta-keys",
nargs="+",
required=True,
help="List of required key in meta part of source.",
)
args = parser.parse_args(argv)
return has_meta_key(paths=args.filenames, meta_keys=args.meta_keys)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
parser.add_argument(
"--tags",
nargs="+",
required=True,
help="A list of tags that source can have.",
)
args = parser.parse_args(argv)
return validate_tags(paths=args.filenames, tags=args.tags)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_catalog_args(parser)
args = parser.parse_args(argv)
try:
catalog = get_json(args.catalog)
except JsonOpenError as e:
print(f"Unable to load catalog file ({e})")
return 1
return check_source_columns(paths=args.filenames, catalog=catalog)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return has_description(paths=args.filenames, manifest=manifest)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
args = parser.parse_args(argv)
status_code = 0
for filename in args.filenames:
with open(filename, "rb+") as file_obj:
status_code_file = check_semicolon(file_obj, replace=True)
if status_code_file:
print(f"Replacing semicolon in {filename}.")
status_code = status_code_file
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
group = parser.add_mutually_exclusive_group()
group.add_argument(
"--ignore",
nargs="*",
help=
"Columns for which do not check whether have a different description.",
)
args = parser.parse_args(argv)
return replace_column_desc(paths=args.filenames, ignore=args.ignore)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
status_code, _, _ = check_refs_sources(paths=args.filenames,
manifest=manifest)
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
parser.add_argument(
"--freshness",
nargs="+",
required=True,
choices=["warn_after", "error_after"],
help="List of required freshness options.",
)
args = parser.parse_args(argv)
return has_freshness(paths=args.filenames,
required_freshness=set(args.freshness))
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_dbt_cmd_args(parser)
add_dbt_cmd_model_args(parser)
args = parser.parse_args(argv)
cmd = prepare_cmd(
args.filenames,
args.global_flags,
args.cmd_flags,
args.model_prefix,
args.model_postfix,
)
return run_dbt_cmd(cmd)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
args = parser.parse_args(argv)
status_code = 0
for filename in args.filenames:
# Read as binary so we can read byte-by-byte
with open(filename, "rb+") as file_obj:
status_code_file = check_semicolon(file_obj)
if status_code_file:
print(f"{filename}: contains a semicolon at the end. "
f"dbt does not support that.")
status_code = status_code_file
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
add_catalog_args(parser)
parser.add_argument(
"--properties-file",
required=True,
type=str,
help="""Location of file where new model properties should
be generated. Suffix has to be `yml` or `yaml`. It can also include
{database}, {schema}, {name} and {alias} variables.
E.g. /models/{schema}/{name}.yml for model `foo.bar` will create
properties file in /models/foo/bar.yml.
""",
)
args = parser.parse_args(argv)
if not Path(args.properties_file).suffix in [".yml", ".yaml"]:
print(
"Input parameter `--schema-file` has to"
" contain `.yml` or `.yaml` extension."
)
return 1
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
try:
catalog = get_json(args.catalog)
except JsonOpenError as e:
print(f"Unable to load catalog file ({e})")
return 1
status_code = generate_properties_file(
paths=args.filenames,
manifest=manifest,
catalog=catalog,
properties_file=args.properties_file,
)
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--tests",
metavar="KEY=VALUE",
nargs="+",
required=True,
help="Set a number of key-value pairs."
" Key is type of test (data or schema) and value is required "
"minimal number of tests eg. --test data=1 schema=2"
"(do not put spaces before or after the = sign). "
"",
action=ParseDict,
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
required_tests = {}
for test_type, cnt in args.tests.items():
allowed_types = ["data", "schema"]
if test_type not in allowed_types:
parser.error(
f"Test type {test_type} is not one of allowed types {allowed_types}"
)
try:
test_cnt = int(cnt)
except ValueError:
parser.error(f"Unable to cast {cnt} to int.")
required_tests[test_type] = test_cnt
return check_test_cnt(
paths=args.filenames, manifest=manifest, required_tests=required_tests
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
args = parser.parse_args(argv)
status_code = 0
for filename in args.filenames:
sql = Path(filename).read_text()
status_code_file, tables = has_table_name(sql, filename)
if status_code_file:
result = "\n- ".join(list(tables)) # pragma: no mutate
print(
f"{filename}: "
f"does not use source() or ref() macros for tables:\n- {result}",
)
status_code = status_code_file
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--min-child-cnt",
type=int,
default=0,
help="Minimal number of child relations.",
)
parser.add_argument(
"--max-child-cnt",
type=int,
help="Miximal number of child relations.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
required_cnt = [
{
"operator": operator.lt,
"type": "min",
"dep": "childs",
"cnt": args.min_child_cnt,
},
{
"operator": operator.gt,
"type": "max",
"dep": "childs",
"cnt": args.max_child_cnt,
},
]
return check_child_parent_cnt(paths=args.filenames,
manifest=manifest,
required_cnt=required_cnt)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
white_black = parser.add_mutually_exclusive_group()
white_black.add_argument(
"--whitelist",
nargs="+",
help="Whitelisted schemas.",
)
white_black.add_argument(
"--blacklist",
nargs="+",
help="Blacklisted schemas.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
if not (args.blacklist or args.whitelist):
print(
"Please specify at least one `--blacklist` or `--whitelist` option."
)
return 1
return check_parents_schema(
paths=args.filenames,
manifest=manifest,
blacklist=args.blacklist,
whitelist=args.whitelist,
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--tags",
nargs="+",
required=True,
help="A list of tags that models can have.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return validate_tags(paths=args.filenames,
manifest=manifest,
tags=args.tags)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--meta-keys",
nargs="+",
required=True,
help="List of required key in meta part of model.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return has_meta_key(paths=args.filenames,
manifest=manifest,
meta_keys=args.meta_keys)
