def build_project( self, selectors: Optional[List[str]] = None, exclusions: Optional[List[str]] = None, ) -> None: """Creates an object representation of the project's LookML. Args: selectors: List of selector strings in 'model_name/explore_name' format. The '*' wildcard selects all models or explores. For instance, 'model_name/*' would select all explores in the 'model_name' model. """ # Assign default values for selectors and exclusions if selectors is None: selectors = ["*/*"] if exclusions is None: exclusions = [] logger.info( f"Building LookML project hierarchy for project {self.project.name}" ) all_models = [ Model.from_json(model) for model in self.client.get_lookml_models() ] project_models = [ model for model in all_models if model.project_name == self.project.name ] if not project_models: raise LookMlNotFound( name="project-models-not-found", title= "No matching models found for the specified project and selectors.", detail=(f"Go to {self.client.base_url}/projects and confirm " "a) at least one model exists for the project and " "b) it has an active configuration."), ) for model in project_models: model.explores = [ explore for explore in model.explores if is_selected(model.name, explore.name, selectors, exclusions) ] for explore in model.explores: dimensions_json = self.client.get_lookml_dimensions( model.name, explore.name) for dimension_json in dimensions_json: dimension = Dimension.from_json(dimension_json, model.name, explore.name) dimension.url = self.client.base_url + dimension.url if not dimension.ignore: explore.add_dimension(dimension) self.project.models = [ model for model in project_models if len(model.explores) > 0 ]
def test_exclude_exact_model_and_explore_should_not_match(): assert not is_selected("model_a", "explore_a", ["*/*"], ["model_a/explore_a"])
def test_exclude_explore_wildcard_should_not_match(): assert not is_selected("model_a", "explore_a", ["*/*"], ["*/explore_a"])
def test_select_wrong_explore_should_not_match(): assert not is_selected("model_a", "explore_a", ["model_a/explore_b"], [])
def test_select_exact_model_and_explore_should_match(): assert is_selected("model_a", "explore_a", ["model_a/explore_a"], [])
def test_select_explore_wildcard_should_match(): assert is_selected("model_a", "explore_a", ["*/explore_a"], []) assert is_selected("model_b", "explore_a", ["*/explore_a"], [])
def validate( self, selectors: Optional[List[str]] = None, exclusions: Optional[List[str]] = None, ) -> Dict[str, Any]: # Assign default values for selectors and exclusions if selectors is None: selectors = ["*/*"] if exclusions is None: exclusions = [] all_tests = self.client.all_lookml_tests(self.project) selected_tests = [] test_to_explore = {} for test in all_tests: if is_selected(test["model_name"], test["explore_name"], selectors, exclusions): selected_tests.append(test) # The error objects don't contain the name of the explore # We create this mapping to help look up the explore from the test name test_to_explore[test["name"]] = test["explore_name"] test_count = len(selected_tests) if test_count == 0: raise SpectaclesException( name="no-data-tests-found", title="No data tests found.", detail= ("If you're using --explores or --exclude, make sure your project " "has data tests that reference those models or explores."), ) printer.print_header( f"Running {test_count} {'test' if test_count == 1 else 'tests'}") test_results: List[Dict[str, Any]] = [] for test in selected_tests: test_name = test["name"] model_name = test["model_name"] results = self.client.run_lookml_test(self.project, model=model_name, test=test_name) test_results.extend(results) tested = [] errors = [] for result in test_results: explore = test_to_explore[result["test_name"]] test = { "model": result["model_name"], "explore": explore, "passed": result["success"], } tested.append(test) if not result["success"]: for error in result["errors"]: project, file_path = error["file_path"].split("/", 1) lookml_url = ( f"{self.client.base_url}/projects/{self.project}" f"/files/{file_path}?line={error['line_number']}") errors.append( DataTestError( model=error["model_id"], explore=error["explore"], message=error["message"], test_name=result["test_name"], lookml_url=lookml_url, ).__dict__) def reduce_result(results): """Aggregate individual test results to get pass/fail by explore""" agg = OrderedDict() for result in results: # Keys by model and explore, adds additional values for `passed` to a set agg.setdefault((result["model"], result["explore"]), set()).add(result["passed"]) reduced = [{ "model": k[0], "explore": k[1], "passed": min(v) } for k, v in agg.items()] return reduced tested = reduce_result(tested) for test in tested: printer.print_validation_result( passed=test["passed"], source=f"{test['model']}.{test['explore']}") passed = min((test["passed"] for test in tested), default=True) return { "validator": "data_test", "status": "passed" if passed else "failed", "tested": tested, "errors": errors, }
def test_empty_selector_should_raise_value_error(): with pytest.raises(ValueError): is_selected("model_a", "explore_a", [], [])