def validate_content(
self,
selectors: List[str],
exclusions: List[str],
incremental: bool = False,
exclude_personal: bool = False,
) -> Dict[str, Any]:
with self.branch_manager:
validator = ContentValidator(self.client, self.project,
exclude_personal)
logger.info("Building LookML project hierarchy for project "
f"'{self.project}' @ {self.branch_manager.ref}")
validator.build_project(selectors, exclusions)
explore_count = validator.project.count_explores()
print_header(f"Validating content based on {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'}" +
(" [incremental mode] " if incremental else ""))
results = validator.validate()
if incremental and self.branch_manager.name != "master":
logger.debug("Starting another content validation against master")
self.branch_manager.commit_ref = None
self.branch_manager.name = "master"
with self.branch_manager:
logger.debug("Building LookML project hierarchy for project "
f"'{self.project}' @ {self.branch_manager.ref}")
validator.build_project(selectors, exclusions)
main_results = validator.validate()
return self._incremental_results(main=main_results,
additional=results)
else:
return results
def validate_sql(
self,
branch: Optional[str],
commit: Optional[str],
selectors: List[str],
exclusions: List[str],
mode: QueryMode = "batch",
concurrency: int = 10,
profile: bool = False,
runtime_threshold: int = 5,
) -> Dict[str, Any]:
with self.branch_manager(branch, commit):
validator = SqlValidator(self.client, self.project, concurrency,
runtime_threshold)
logger.info("Building LookML project hierarchy for project "
f"'{self.project}' @ {self.branch_manager.ref}")
validator.build_project(selectors,
exclusions,
build_dimensions=True)
explore_count = validator.project.count_explores()
print_header(f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode] "
f"[concurrency = {validator.query_slots}]")
results = validator.validate(mode, profile)
return results
def validate(self, mode: str = "batch") -> List[SqlError]:
"""Queries selected explores and returns any errors.
Args:
batch: When true, runs one query per explore (using all dimensions). When
false, runs one query per dimension. Batch mode increases query speed
but can only return the first error encountered for each dimension.
Returns:
List[SqlError]: SqlErrors encountered while querying the explore.
"""
explore_count = self._count_explores()
printer.print_header(
f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode]")
errors = self._query(mode)
if mode == "hybrid" and self.project.errored:
errors = self._query(mode)
for model in sorted(self.project.models, key=lambda x: x.name):
for explore in sorted(model.explores, key=lambda x: x.name):
if explore.errored:
logger.info(
f"✗ {printer.red(model.name + '.' + explore.name)} failed"
)
else:
logger.info(
f"✓ {printer.green(model.name + '.' + explore.name)} passed"
)
return errors
def validate_data_tests(self, selectors: List[str],
exclusions: List[str]) -> Dict[str, Any]:
with self.branch_manager:
validator = DataTestValidator(self.client, self.project)
logger.info("Building LookML project hierarchy for project "
f"'{self.project}' @ {self.branch_manager.ref}")
validator.build_project(selectors, exclusions)
explore_count = validator.project.count_explores()
print_header(f"Running data tests based on {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'}")
results = validator.validate()
return results
def validate(self) -> List[DataTestError]:
tests = self.client.all_lookml_tests(self.project)
test_count = len(tests)
printer.print_header(
f"Running {test_count} {'test' if test_count == 1 else 'tests'}")
errors = []
test_results = self.client.run_lookml_test(self.project)
for result in test_results:
if not result["success"]:
for error in result["errors"]:
errors.append(
DataTestError(
path=
f"{result['model_name']}/{result['test_name']}",
message=error["message"],
))
return errors
def validate(self, mode: QueryMode = "batch") -> Dict[str, Any]:
"""Queries selected explores and returns the project tree with errors."""
self._query_by_task_id = {}
explore_count = self._count_explores()
printer.print_header(
f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode] "
f"[concurrency = {self.query_slots}]")
self._create_and_run(mode)
if mode == "hybrid" and self.project.errored:
self._create_and_run(mode)
for model in sorted(self.project.models, key=lambda x: x.name):
for explore in sorted(model.explores, key=lambda x: x.name):
message = f"{model.name}.{explore.name}"
printer.print_validation_result(passed=not explore.errored,
source=message)
return self.project.get_results(mode)
def validate(
self, mode: QueryMode = "batch", profile: bool = False
) -> Dict[str, Any]:
"""Queries selected explores and returns the project tree with errors."""
self._query_by_task_id = {}
self._create_and_run(mode)
if mode == "hybrid" and self.project.errored:
self._create_and_run(mode)
if profile:
char = "."
print_header("Query profiler results", char=char, leading_newline=False)
if self.long_running_queries:
queries_in_order = sorted(
self.long_running_queries, key=lambda x: x[2], reverse=True
) # type: ignore
output = tabulate(
queries_in_order,
headers=[
"Type",
"Name",
"Runtime (s)",
"Query ID",
"Explore From Here",
],
tablefmt="github",
numalign="left",
floatfmt=".1f",
)
else:
output = (
f"All queries completed in less than {self.runtime_threshold} "
"seconds."
)
logger.info(output)
print_header(char, char=char)
return self.project.get_results(validator="sql", mode=mode)
def validate(self, mode: str = "batch") -> List[SqlError]:
"""Queries selected explores and returns any errors.
Args:
batch: When true, runs one query per explore (using all dimensions). When
false, runs one query per dimension. Batch mode increases query speed
but can only return the first error encountered for each dimension.
Returns:
List[SqlError]: SqlErrors encountered while querying the explore.
"""
explore_count = self._count_explores()
printer.print_header(
f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode]")
loop = asyncio.get_event_loop()
signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)
for s in signals:
loop.add_signal_handler(
s, lambda s=s: asyncio.create_task(self.shutdown(s, loop)))
errors = list(loop.run_until_complete(self._query(mode)))
if mode == "hybrid" and self.project.errored:
errors = list(loop.run_until_complete(self._query(mode)))
for model in sorted(self.project.models, key=lambda x: x.name):
for explore in sorted(model.explores, key=lambda x: x.name):
message = f"{model.name}.{explore.name}"
if explore.errored:
printer.print_validation_result("error", message)
else:
printer.print_validation_result("success", message)
return errors
def validate(
self,
selectors: Optional[List[str]] = None,
exclusions: Optional[List[str]] = None,
) -> Dict[str, Any]:
# Assign default values for selectors and exclusions
if selectors is None:
selectors = ["*/*"]
if exclusions is None:
exclusions = []
all_tests = self.client.all_lookml_tests(self.project)
selected_tests = []
test_to_explore = {}
for test in all_tests:
if is_selected(test["model_name"], test["explore_name"], selectors,
exclusions):
selected_tests.append(test)
# The error objects don't contain the name of the explore
# We create this mapping to help look up the explore from the test name
test_to_explore[test["name"]] = test["explore_name"]
test_count = len(selected_tests)
if test_count == 0:
raise SpectaclesException(
name="no-data-tests-found",
title="No data tests found.",
detail=
("If you're using --explores or --exclude, make sure your project "
"has data tests that reference those models or explores."),
)
printer.print_header(
f"Running {test_count} {'test' if test_count == 1 else 'tests'}")
test_results: List[Dict[str, Any]] = []
for test in selected_tests:
test_name = test["name"]
model_name = test["model_name"]
results = self.client.run_lookml_test(self.project,
model=model_name,
test=test_name)
test_results.extend(results)
tested = []
errors = []
for result in test_results:
explore = test_to_explore[result["test_name"]]
test = {
"model": result["model_name"],
"explore": explore,
"passed": result["success"],
}
tested.append(test)
if not result["success"]:
for error in result["errors"]:
project, file_path = error["file_path"].split("/", 1)
lookml_url = (
f"{self.client.base_url}/projects/{self.project}"
f"/files/{file_path}?line={error['line_number']}")
errors.append(
DataTestError(
model=error["model_id"],
explore=error["explore"],
message=error["message"],
test_name=result["test_name"],
lookml_url=lookml_url,
).__dict__)
def reduce_result(results):
"""Aggregate individual test results to get pass/fail by explore"""
agg = OrderedDict()
for result in results:
# Keys by model and explore, adds additional values for `passed` to a set
agg.setdefault((result["model"], result["explore"]),
set()).add(result["passed"])
reduced = [{
"model": k[0],
"explore": k[1],
"passed": min(v)
} for k, v in agg.items()]
return reduced
tested = reduce_result(tested)
for test in tested:
printer.print_validation_result(
passed=test["passed"],
source=f"{test['model']}.{test['explore']}")
passed = min((test["passed"] for test in tested), default=True)
return {
"validator": "data_test",
"status": "passed" if passed else "failed",
"tested": tested,
"errors": errors,
}
