def run_content(
project,
branch,
explores,
excludes,
base_url,
client_id,
client_secret,
port,
api_version,
remote_reset,
import_projects,
commit_ref,
incremental,
exclude_personal,
) -> None:
runner = Runner(
base_url,
project,
branch,
client_id,
client_secret,
port,
api_version,
remote_reset,
import_projects,
commit_ref,
)
results = runner.validate_content(explores, excludes, incremental,
exclude_personal)
for test in sorted(results["tested"],
key=lambda x: (x["model"], x["explore"])):
message = f"{test['model']}.{test['explore']}"
printer.print_validation_result(passed=test["passed"], source=message)
errors = sorted(
results["errors"],
key=lambda x: (x["model"], x["explore"], x["metadata"]["field_name"]),
)
if errors:
for error in errors:
printer.print_content_error(
error["model"],
error["explore"],
error["message"],
error["metadata"]["content_type"],
error["metadata"]["space"],
error["metadata"]["title"],
error["metadata"]["url"],
)
logger.info("")
raise GenericValidationError
else:
logger.info("")
def validate(self, mode: QueryMode = "batch") -> Dict[str, Any]:
"""Queries selected explores and returns the project tree with errors."""
self._query_by_task_id = {}
explore_count = self._count_explores()
printer.print_header(
f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode] "
f"[concurrency = {self.query_slots}]")
self._create_and_run(mode)
if mode == "hybrid" and self.project.errored:
self._create_and_run(mode)
for model in sorted(self.project.models, key=lambda x: x.name):
for explore in sorted(model.explores, key=lambda x: x.name):
message = f"{model.name}.{explore.name}"
printer.print_validation_result(passed=not explore.errored,
source=message)
return self.project.get_results(mode)
def validate(self) -> List[DataTestError]:
tests = self.client.all_lookml_tests(self.project)
test_count = len(tests)
printer.print_header(
f"Running {test_count} {'test' if test_count == 1 else 'tests'}")
errors = []
test_results = self.client.run_lookml_test(self.project)
for result in test_results:
message = f"{result['model_name']}.{result['test_name']}"
if result["success"]:
printer.print_validation_result("success", message)
else:
for error in result["errors"]:
printer.print_validation_result("error", message)
errors.append(
DataTestError(
path=
f"{result['model_name']}/{result['test_name']}",
message=error["message"],
))
return errors
def validate(self, mode: str = "batch") -> List[SqlError]:
"""Queries selected explores and returns any errors.
Args:
batch: When true, runs one query per explore (using all dimensions). When
false, runs one query per dimension. Batch mode increases query speed
but can only return the first error encountered for each dimension.
Returns:
List[SqlError]: SqlErrors encountered while querying the explore.
"""
explore_count = self._count_explores()
printer.print_header(
f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode]")
loop = asyncio.get_event_loop()
signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)
for s in signals:
loop.add_signal_handler(
s, lambda s=s: asyncio.create_task(self.shutdown(s, loop)))
errors = list(loop.run_until_complete(self._query(mode)))
if mode == "hybrid" and self.project.errored:
errors = list(loop.run_until_complete(self._query(mode)))
for model in sorted(self.project.models, key=lambda x: x.name):
for explore in sorted(model.explores, key=lambda x: x.name):
message = f"{model.name}.{explore.name}"
if explore.errored:
printer.print_validation_result("error", message)
else:
printer.print_validation_result("success", message)
return errors
def validate(
self,
selectors: Optional[List[str]] = None,
exclusions: Optional[List[str]] = None,
) -> Dict[str, Any]:
# Assign default values for selectors and exclusions
if selectors is None:
selectors = ["*/*"]
if exclusions is None:
exclusions = []
all_tests = self.client.all_lookml_tests(self.project)
selected_tests = []
test_to_explore = {}
for test in all_tests:
if is_selected(test["model_name"], test["explore_name"], selectors,
exclusions):
selected_tests.append(test)
# The error objects don't contain the name of the explore
# We create this mapping to help look up the explore from the test name
test_to_explore[test["name"]] = test["explore_name"]
test_count = len(selected_tests)
if test_count == 0:
raise SpectaclesException(
name="no-data-tests-found",
title="No data tests found.",
detail=
("If you're using --explores or --exclude, make sure your project "
"has data tests that reference those models or explores."),
)
printer.print_header(
f"Running {test_count} {'test' if test_count == 1 else 'tests'}")
test_results: List[Dict[str, Any]] = []
for test in selected_tests:
test_name = test["name"]
model_name = test["model_name"]
results = self.client.run_lookml_test(self.project,
model=model_name,
test=test_name)
test_results.extend(results)
tested = []
errors = []
for result in test_results:
explore = test_to_explore[result["test_name"]]
test = {
"model": result["model_name"],
"explore": explore,
"passed": result["success"],
}
tested.append(test)
if not result["success"]:
for error in result["errors"]:
project, file_path = error["file_path"].split("/", 1)
lookml_url = (
f"{self.client.base_url}/projects/{self.project}"
f"/files/{file_path}?line={error['line_number']}")
errors.append(
DataTestError(
model=error["model_id"],
explore=error["explore"],
message=error["message"],
test_name=result["test_name"],
lookml_url=lookml_url,
).__dict__)
def reduce_result(results):
"""Aggregate individual test results to get pass/fail by explore"""
agg = OrderedDict()
for result in results:
# Keys by model and explore, adds additional values for `passed` to a set
agg.setdefault((result["model"], result["explore"]),
set()).add(result["passed"])
reduced = [{
"model": k[0],
"explore": k[1],
"passed": min(v)
} for k, v in agg.items()]
return reduced
tested = reduce_result(tested)
for test in tested:
printer.print_validation_result(
passed=test["passed"],
source=f"{test['model']}.{test['explore']}")
passed = min((test["passed"] for test in tested), default=True)
return {
"validator": "data_test",
"status": "passed" if passed else "failed",
"tested": tested,
"errors": errors,
}
def run_sql(
log_dir,
project,
branch,
explores,
exclude,
base_url,
client_id,
client_secret,
port,
api_version,
mode,
remote_reset,
import_projects,
concurrency,
commit_ref,
) -> None:
"""Runs and validates the SQL for each selected LookML dimension."""
runner = Runner(
base_url,
project,
branch,
client_id,
client_secret,
port,
api_version,
remote_reset,
import_projects,
commit_ref,
)
def iter_errors(lookml: List) -> Iterable:
for item in lookml:
if item.errored:
yield item
results = runner.validate_sql(explores, exclude, mode, concurrency)
for test in sorted(results["tested"],
key=lambda x: (x["model"], x["explore"])):
message = f"{test['model']}.{test['explore']}"
printer.print_validation_result(passed=test["passed"], source=message)
errors = sorted(
results["errors"],
key=lambda x:
(x["model"], x["explore"], x["metadata"].get("dimension")),
)
if errors:
for error in errors:
printer.print_sql_error(
model=error["model"],
explore=error["explore"],
message=error["message"],
sql=error["metadata"]["sql"],
log_dir=log_dir,
dimension=error["metadata"].get("dimension"),
lookml_url=error["metadata"].get("lookml_url"),
)
if mode == "batch":
logger.info(
printer.dim(
"\n\nTo determine the exact dimensions responsible for "
f"{'this error' if len(errors) == 1 else 'these errors'}, "
"you can re-run \nSpectacles in single-dimension mode, "
"with `--mode single`.\n\nYou can also run this original "
"validation with `--mode hybrid` to do this automatically."
))
logger.info("")
raise GenericValidationError
else:
logger.info("")
def test_print_validation_result_should_work():
printer.print_validation_result(passed=True, source="model.explore")
printer.print_validation_result(passed=False, source="model.explore")