def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--tests",
nargs="+",
required=True,
help="List of acceptable tests.",
)
parser.add_argument(
"--test-cnt",
type=int,
default=1,
help="Minimum number of tests required.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return check_test_cnt(
paths=args.filenames,
manifest=manifest,
test_group=args.tests,
test_cnt=args.test_cnt,
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
status_code = 0
for filename in args.filenames:
file = Path(filename)
sql = file.read_text()
status_code_file, tables = has_table_name(sql, filename)
if status_code_file:
status_code = status_code_file
to_replace = itertools.chain(
get_ref_from_name(manifest, tables),
get_source_from_name(manifest, tables),
get_unknown_source(tables),
)
for replacement in to_replace:
sql = re.sub(*replacement, sql, re.IGNORECASE)
file.write_text(sql, encoding="utf-8")
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--schema-file",
required=True,
type=str,
help="""Location of schema.yml file. Where new source tables should
be created.
""",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
_, _, sources = check_refs_sources(paths=args.filenames, manifest=manifest)
status_code = create_missing_sources(sources, output_path=args.schema_file)
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
add_catalog_args(parser)
parser.add_argument(
"--properties-file",
required=True,
type=str,
help="""Location of file where new model properties should
be generated. Suffix has to be `yml` or `yaml`. It can also include
{database}, {schema}, {name} and {alias} variables.
E.g. /models/{schema}/{name}.yml for model `foo.bar` will create
properties file in /models/foo/bar.yml.
""",
)
args = parser.parse_args(argv)
if not Path(args.properties_file).suffix in [".yml", ".yaml"]:
print(
"Input parameter `--schema-file` has to"
" contain `.yml` or `.yaml` extension."
)
return 1
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
try:
catalog = get_json(args.catalog)
except JsonOpenError as e:
print(f"Unable to load catalog file ({e})")
return 1
status_code = generate_properties_file(
paths=args.filenames,
manifest=manifest,
catalog=catalog,
properties_file=args.properties_file,
)
return status_code
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_catalog_args(parser)
args = parser.parse_args(argv)
try:
catalog = get_json(args.catalog)
except JsonOpenError as e:
print(f"Unable to load catalog file ({e})")
return 1
return check_source_columns(paths=args.filenames, catalog=catalog)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return has_description(paths=args.filenames, manifest=manifest)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
status_code, _, _ = check_refs_sources(paths=args.filenames,
manifest=manifest)
return status_code
def test_check_script_ref_and_source(
input_s,
expected_status_code,
missing_models,
missing_source,
manifest_path_str,
tmpdir,
):
path = tmpdir.join("file.sql")
path.write_text(input_s, "utf-8")
manifest = get_json(manifest_path_str)
ret, models, sources = check_refs_sources(paths=[path], manifest=manifest)
assert ret == expected_status_code
assert models == missing_models
assert sources == missing_source
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--tests",
metavar="KEY=VALUE",
nargs="+",
required=True,
help="Set a number of key-value pairs."
" Key is type of test (data or schema) and value is required "
"minimal number of tests eg. --test data=1 schema=2"
"(do not put spaces before or after the = sign). "
"",
action=ParseDict,
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
required_tests = {}
for test_type, cnt in args.tests.items():
allowed_types = ["data", "schema"]
if test_type not in allowed_types:
parser.error(
f"Test type {test_type} is not one of allowed types {allowed_types}"
)
try:
test_cnt = int(cnt)
except ValueError:
parser.error(f"Unable to cast {cnt} to int.")
required_tests[test_type] = test_cnt
return check_test_cnt(
paths=args.filenames, manifest=manifest, required_tests=required_tests
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--min-child-cnt",
type=int,
default=0,
help="Minimal number of child relations.",
)
parser.add_argument(
"--max-child-cnt",
type=int,
help="Miximal number of child relations.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
required_cnt = [
{
"operator": operator.lt,
"type": "min",
"dep": "childs",
"cnt": args.min_child_cnt,
},
{
"operator": operator.gt,
"type": "max",
"dep": "childs",
"cnt": args.max_child_cnt,
},
]
return check_child_parent_cnt(paths=args.filenames,
manifest=manifest,
required_cnt=required_cnt)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
white_black = parser.add_mutually_exclusive_group()
white_black.add_argument(
"--whitelist",
nargs="+",
help="Whitelisted schemas.",
)
white_black.add_argument(
"--blacklist",
nargs="+",
help="Blacklisted schemas.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
if not (args.blacklist or args.whitelist):
print(
"Please specify at least one `--blacklist` or `--whitelist` option."
)
return 1
return check_parents_schema(
paths=args.filenames,
manifest=manifest,
blacklist=args.blacklist,
whitelist=args.whitelist,
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--tags",
nargs="+",
required=True,
help="A list of tags that models can have.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return validate_tags(paths=args.filenames,
manifest=manifest,
tags=args.tags)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
add_filenames_args(parser)
add_manifest_args(parser)
parser.add_argument(
"--meta-keys",
nargs="+",
required=True,
help="List of required key in meta part of model.",
)
args = parser.parse_args(argv)
try:
manifest = get_json(args.manifest)
except JsonOpenError as e:
print(f"Unable to load manifest file ({e})")
return 1
return has_meta_key(paths=args.filenames,
manifest=manifest,
meta_keys=args.meta_keys)
