def test_sub_local_routines(self):
data_dir = TEST_DIR / "data" / "test_sql" / "moz-fx-data-test-project"
raw_routines = parse_routine.read_routine_dir(data_dir / "udf")
raw_routines.update(
parse_routine.read_routine_dir(data_dir / "procedure"))
raw_routine = parse_routine.RawRoutine.from_file(
data_dir / "udf" / "test_shift_28_bits_one_day" /
"udf.sql").tests[0]
assert "CREATE TEMP FUNCTION" not in raw_routine
assert "CREATE TEMP FUNCTION udf_test_bitmask_lowest_28" not in raw_routine
result = parse_routine.sub_local_routines(raw_routine,
self.udf_dir.parent,
raw_routines)
assert "CREATE TEMP FUNCTION udf_test_shift_28_bits_one_day" in result
assert "CREATE TEMP FUNCTION udf_test_bitmask_lowest_28" in result
text = "SELECT udf.test_bitmask_lowest_28(23), mozfun.hist.extract('{}')"
result = parse_routine.sub_local_routines(text, self.udf_dir.parent,
raw_routines)
assert "CREATE TEMP FUNCTION udf_test_bitmask_lowest_28" in result
# There is no defn for hist.extract in the `raw_routines`,
# so we expect this to be unreplaced
assert "hist_extract" not in result
assert "mozfun.hist.extract" in result
text = "CALL procedure.test_procedure(23);"
result = parse_routine.sub_local_routines(text,
self.udf_dir.parent,
raw_routines,
stored_procedure_test=True)
assert (
"CREATE OR REPLACE PROCEDURE\n _generic_dataset_.procedure_test_procedure"
in result)
def validate(project_dirs):
"""Validate UDF docs."""
is_valid = True
for project_dir in project_dirs:
if os.path.isdir(project_dir):
parsed_routines = read_routine_dir(project_dir)
for root, dirs, files in os.walk(project_dir):
if os.path.basename(root) == EXAMPLE_DIR:
for file in files:
dry_run_sql = sub_local_routines(
(Path(root) / file).read_text(),
project_dir,
parsed_routines,
)
# store sql in temporary file for dry_run
tmp_dir = Path(tempfile.mkdtemp()) / Path(root)
tmp_dir.mkdir(parents=True, exist_ok=True)
tmp_example_file = tmp_dir / file
tmp_example_file.write_text(dry_run_sql)
if not DryRun(str(tmp_example_file)).is_valid():
is_valid = False
if not is_valid:
print("Invalid examples.")
sys.exit(1)
def sql_for_dry_run(file, parsed_routines, project_dir):
"""
Return the example SQL used for the dry run.
Injects all UDFs the example depends on as temporary functions.
"""
dry_run_sql = ""
with open(file) as sql:
example_sql = sql.read()
# add UDFs that example depends on as temporary functions
for udf, raw_routine in parsed_routines.items():
if udf in example_sql:
query = "".join(raw_routine.definitions)
dry_run_sql += sub_local_routines(query, project_dir,
parsed_routines)
dry_run_sql += example_sql
for udf, _ in parsed_routines.items():
# temporary UDFs cannot contain dots, rename UDFS
dry_run_sql = dry_run_sql.replace(udf, udf.replace(".", "_"))
# remove explicit project references
dry_run_sql = dry_run_sql.replace(
os.path.basename(project_dir) + ".", "")
return dry_run_sql
def test_sql_for_dry_run_no_dependencies(self):
file = (
TEST_DIR
/ "data"
/ "test_docs"
/ "generated_docs"
/ "test_dataset1"
/ "examples"
/ "example1.sql"
)
sql = sub_local_routines(file.read_text(), project="", raw_routines={})
assert sql == file.read_text()