def update_branch(self, project: str, branch: str, ref: str = "origin/master"): """Updates a branch to the ref prodvided. Args: project: Name of the Looker project to use. branch: Name of the branch to update. ref: The ref to update the branch from. """ logger.debug( f"Updating branch '{branch}' on project '{project}' to ref '{ref}'" ) body = {"name": branch, "ref": ref} url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.put(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-update-branch", title="Couldn't update Git branch.", status=response.status_code, detail=( f"Unable to update branch '{branch}' " f"in project '{project}' using ref '{ref}'. " "Please try again." ), response=response, )
def create_branch(self, project: str, branch: str, ref: str = "origin/master"): """Creates a branch in the given project. Args: project: Name of the Looker project to use. branch: Name of the branch to create. ref: The ref to create the branch from. """ logger.debug( f"Creating branch '{branch}' on project '{project}' with ref '{ref}'" ) body = {"name": branch, "ref": ref} url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.post(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-create-branch", title="Couldn't create new Git branch.", status=response.status_code, detail=( f"Unable to create branch '{branch}' " f"in project '{project}' using ref '{ref}'. " "Confirm the branch doesn't already exist and try again." ), response=response, )
def reset_to_remote(self, project: str) -> None: """Reset a project development branch to the revision of the project that is on the remote. Args: project: Name of the Looker project to use. """ logger.debug(f"Resetting branch to remote.") url = utils.compose_url( self.api_url, path=["projects", project, "reset_to_remote"] ) response = self.post(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-reset-remote", title="Couldn't checkout Git branch.", status=response.status_code, detail=( f"Unable to reset local Git branch" "to match remote. Please try again." ), response=response, )
def get_all_branches(self, project: str) -> List[str]: """Returns a list of git branches in the project repository. Args: project: Name of the Looker project to use. """ logger.debug(f"Getting all Git branches in project '{project}'") url = utils.compose_url( self.api_url, path=["projects", project, "git_branches"] ) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-branches", title="Couldn't get all Git branches.", status=response.status_code, detail=( f"Unable to get all Git branches in project '{project}'. " "Please try again." ), response=response, ) return [branch["name"] for branch in response.json()]
def delete_branch(self, project: str, branch: str): """Deletes a branch in the given project. Args: project: Name of the Looker project to use. branch: Name of the branch to delete. """ logger.debug(f"Deleting branch '{branch}' in project '{project}'") url = utils.compose_url( self.api_url, path=["projects", project, "git_branch", branch] ) response = self.delete(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-delete-branch", title="Couldn't delete Git branch.", status=response.status_code, detail=( f"Unable to delete branch '{branch}' " f"in project '{project}'. Please try again." ), response=response, )
def all_lookml_tests(self, project: str) -> List[JsonDict]: """Gets all LookML/data tests for a given project. Args: project: Name of the Looker project to use Returns: List[JsonDict]: JSON response containing all LookML/data tests """ logger.debug(f"Getting LookML tests for project {project}") url = utils.compose_url(self.api_url, path=["projects", project, "lookml_tests"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-data-tests", title="Couldn't retrieve all data tests.", status=response.status_code, detail=(f"Unable to retrieve all data tests for " f"project '{project}'. Please try again."), response=response, ) return response.json()
async def get_query_task_multi_results( self, session: aiohttp.ClientSession, query_task_ids: List[str] ) -> JsonDict: """Returns query task results. If a ClientError or TimeoutError is received, attempts to retry. Args: query_task_ids: IDs for the query tasks running asynchronously. Returns: List[JsonDict]: JSON response from the query task. """ # Using old-style string formatting so that strings are formatted lazily logger.debug( "Attempting to get results for %d query tasks", len(query_task_ids) ) url = utils.compose_url(self.api_url, path=["query_tasks", "multi_results"]) async with session.get( url=url, params={"query_task_ids": ",".join(query_task_ids)} ) as response: result = await response.json() response.raise_for_status() return result
def wrapper(*args, **kwargs): try: return function(*args, **kwargs) except ValidationError as error: sys.exit(error.exit_code) except SpectaclesException as error: logger.error( f"{error}\n\n" + printer.dim( "For support, please create an issue at " "https://github.com/spectacles-ci/spectacles/issues" ) + "\n" ) sys.exit(error.exit_code) except Exception as error: logger.debug(error, exc_info=True) logger.error( f'Encountered unexpected {error.__class__.__name__}: "{error}"\n' f"Full error traceback logged to {LOG_FILEPATH}\n\n" + printer.dim( "For support, please create an issue at " "https://github.com/spectacles-ci/spectacles/issues" ) + "\n" ) sys.exit(1)
def get_lookml_models(self, fields: List = []) -> List[JsonDict]: """Gets all models and explores from the LookmlModel endpoint. Returns: List[JsonDict]: JSON response containing LookML models and explores. """ logger.debug(f"Getting all models and explores from {self.base_url}") params = {} if fields: params["fields"] = fields url = utils.compose_url(self.api_url, path=["lookml_models"], params=params) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-lookml", title="Couldn't retrieve models and explores.", status=response.status_code, detail="Unable to retrieve LookML details. Please try again.", response=response, ) return response.json()
def _get_query_results(self, query_task_ids: List[str]) -> List[QueryResult]: """Returns ID, status, and error message for all query tasks""" query_results = [] results = self.client.get_query_task_multi_results(query_task_ids) for query_task_id, result in results.items(): status = result["status"] if status not in ("complete", "error", "running", "added", "expired"): raise SpectaclesException( name="unexpected-query-result-status", title="Encountered an unexpected query result status.", detail=(f"Query result status '{status}' was returned " "by the Looker API."), ) logger.debug(f"Query task {query_task_id} status is: {status}") query_result = QueryResult(query_task_id, status) if status == "error": try: error_details = self._extract_error_details(result) except (KeyError, TypeError, IndexError) as error: logger.debug( f"Exiting because of unexpected query result format: {result}" ) raise SpectaclesException( name="unexpected-query-result-format", title="Encountered an unexpected query result format.", detail= f"Unable to extract error details. The unexpected result has been logged.", ) from error else: query_result.error = error_details query_results.append(query_result) return query_results
def hard_reset_branch(self, project: str, branch: str, ref: str): """Hard resets a branch to the ref prodvided. DANGER: hard reset will be force pushed to the remote. Unsaved changes and commits may be permanently lost. Args: project: Name of the Looker project to use. branch: Name of the branch to update. ref: The ref to update the branch from. """ logger.debug( f"Hard resetting branch '{branch}' on project '{project}' to ref '{ref}'" ) body = {"name": branch, "ref": ref} url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.put(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-update-branch", title="Couldn't update Git branch.", status=response.status_code, detail=(f"Unable to update branch '{branch}' " f"in project '{project}' using ref '{ref}'. " "Please try again."), response=response, )
def update_workspace(self, project: str, workspace: str) -> None: """Updates the session workspace. Args: project: Name of the Looker project to use. workspace: The workspace to switch to, either 'production' or 'dev' """ logger.debug(f"Updating session to use the {workspace} workspace") url = utils.compose_url(self.api_url, path=["session"]) body = {"workspace_id": workspace} response = self.patch(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-update-workspace", title="Couldn't update the session's workspace.", status=response.status_code, detail=( f"Unable to update workspace to '{workspace}'. " "If you have any unsaved work on the branch " "checked out by the user whose API credentials " "Spectacles is using, please save it and try again." ), response=response, )
def get_manifest(self, project: str) -> JsonDict: """Gets all the dependent LookML projects defined in the manifest file. Args: project: Name of the Looker project to use. Returns: List[JsonDict]: JSON response containing all dependent projects """ logger.debug("Getting manifest details") url = utils.compose_url(self.api_url, path=["projects", project, "manifest"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-manifest", title="Couldn't retrieve project manifest.", status=response.status_code, detail= (f"Failed to retrieve manifest for project '{project}'. " "Make sure you have a 'manifest.lkml' file in your project, " "then try again."), response=response, ) manifest = response.json() return manifest
def get_active_branch(self, project: str) -> JsonDict: """Gets the active branch for the user in the given project. Args: project: Name of the Looker project to use. Returns: str: Name of the active branch """ logger.debug(f"Getting active branch for project '{project}'") url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-active-branch", title="Couldn't determine active Git branch.", status=response.status_code, detail=( f"Unable to get active branch for project '{project}'. " "Please check that the project exists and try again."), response=response, ) branch_name = response.json()["name"] logger.debug(f"The active branch is '{branch_name}'") return response.json()
def checkout_branch(self, project: str, branch: str) -> None: """Checks out a new git branch. Only works in dev workspace. Args: project: Name of the Looker project to use. branch: Name of the Git branch to check out. """ logger.debug(f"Setting project '{project}' branch to '{branch}'") url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) body = {"name": branch} response = self.put(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-checkout-branch", title="Couldn't checkout Git branch.", status=response.status_code, detail=( f"Unable to checkout Git branch '{branch}'. " "If you have uncommitted changes on the current branch, " "please commit or revert them, then try again."), response=response, )
def get_looker_release_version(self) -> str: """Gets the version number of connected Looker instance. Returns: str: Looker instance release version number (e.g. 6.22.12) """ logger.debug("Checking Looker instance release version") url = utils.compose_url(self.api_url, path=["versions"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-version", title="Couldn't get Looker's release version.", status=response.status_code, detail= ("Unable to get the release version of your Looker instance. " "Please try again."), response=response, ) return response.json()["looker_release_version"]
def request(self, method: str, url: str, *args, **kwargs) -> requests.Response: if self.access_token and self.access_token.expired: logger.debug( "Looker API access token has expired, requesting a new one") self.authenticate() return self.session.request(method, url, *args, **kwargs)
def authenticate(self, client_id: str, client_secret: str, api_version: float) -> None: """Logs in to Looker's API using a client ID/secret pair and an API version. Args: client_id: Looker API client ID. client_secret: Looker API client secret. api_version: Desired API version to use for requests. """ logger.debug("Authenticating Looker API credentials") url = utils.compose_url(self.api_url, path=["login"]) body = {"client_id": client_id, "client_secret": client_secret} response = self.session.post(url=url, data=body) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f"Failed to authenticate to {url}\n" f"Attempted authentication with client ID {client_id}\n" f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"') access_token = response.json()["access_token"] self.session.headers = {"Authorization": f"token {access_token}"} logger.info(f"Connected using Looker API {api_version}")
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]: """Gets all dimensions for an explore from the LookmlModel endpoint. Args: model: Name of LookML model to query. explore: Name of LookML explore to query. Returns: List[str]: Names of all the dimensions in the specified explore. Dimension names are returned in the format 'explore_name.dimension_name'. """ logger.debug(f"Getting all dimensions from explore {explore}") params = {"fields": ["fields"]} url = utils.compose_url( self.api_url, path=["lookml_models", model, "explores", explore], params=params, ) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-dimension-lookml", title="Couldn't retrieve dimensions.", status=response.status_code, detail=("Unable to retrieve dimension LookML details " f"for explore '{model}/{explore}'. Please try again."), response=response, ) return response.json()["fields"]["dimensions"]
def get_query_task_multi_results(self, query_task_ids: List[str]) -> JsonDict: """Returns query task results. If a ClientError or TimeoutError is received, attempts to retry. Args: query_task_ids: IDs for the query tasks running asynchronously. Returns: List[JsonDict]: JSON response from the query task. """ # Using old-style string formatting so that strings are formatted lazily logger.debug( "Attempting to get results for %d query tasks", len(query_task_ids) ) url = utils.compose_url(self.api_url, path=["query_tasks", "multi_results"]) response = self.session.get( url=url, params={"query_task_ids": ",".join(query_task_ids)} ) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"' ) return response.json()
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]: """Gets all dimensions for an explore from the LookmlModel endpoint. Args: model: Name of LookML model to query. explore: Name of LookML explore to query. Returns: List[str]: Names of all the dimensions in the specified explore. Dimension names are returned in the format 'explore_name.dimension_name'. """ logger.debug(f"Getting all dimensions from explore {explore}") url = utils.compose_url( self.api_url, path=["lookml_models", model, "explores", explore] ) response = self.session.get(url=url) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f'Unable to get dimensions for explore "{explore}".\n' f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"' ) return response.json()["fields"]["dimensions"]
async def create_query_task( self, session: aiohttp.ClientSession, query_id: int ) -> str: """Runs a previously created query asynchronously and returns the query task ID. If a ClientError or TimeoutError is received, attempts to retry. Args: session: Existing asychronous HTTP session. query_id: ID of a previously created query to run. Returns: str: ID for the query task, used to check on the status of the query, which is being run asynchronously. """ # Using old-style string formatting so that strings are formatted lazily logger.debug("Starting query %d", query_id) body = {"query_id": query_id, "result_format": "json_detail"} url = utils.compose_url(self.api_url, path=["query_tasks"]) async with session.post( url=url, json=body, params={"cache": "false"} ) as response: result = await response.json() response.raise_for_status() query_task_id = result["id"] logger.debug("Query %d is running under query task %s", query_id, query_task_id) return query_task_id
def run_lookml_test(self, project: str, model: str = None) -> List[JsonDict]: """Runs all LookML/data tests for a given project and model (optional) This command only runs tests in production, as the Looker API doesn't currently allow us to run data tests on a specific branch. Args: project: Name of the Looker project to use model: Optional name of the LookML model to restrict testing to Returns: List[JsonDict]: JSON response containing any LookML/data test errors """ logger.debug(f"Running LookML tests for project {project}") url = utils.compose_url( self.api_url, path=["projects", project, "lookml_tests", "run"] ) if model is not None: response = self.session.get(url=url, params={"model": model}) else: response = self.session.get(url=url) try: response.raise_for_status() except requests.exceptions.HTTPError as error: raise ApiConnectionError( f"Failed to run data tests for project {project}\n" f'Error raised: "{error}"' ) return response.json()
def all_lookml_tests(self, project: str) -> List[JsonDict]: """Gets all LookML/data tests for a given project. Args: project: Name of the Looker project to use Returns: List[JsonDict]: JSON response containing all LookML/data tests """ logger.debug(f"Getting LookML tests for project {project}") url = utils.compose_url( self.api_url, path=["projects", project, "lookml_tests"] ) response = self.session.get(url=url) try: response.raise_for_status() except requests.exceptions.HTTPError as error: raise ApiConnectionError( f"Failed to retrieve data tests for project {project}\n" f'Error raised: "{error}"' ) return response.json()
def setup_temp_branch(self, project: str, original_branch: str) -> str: name = "tmp_spectacles_" + time_hash() logger.debug( f"Branch '{name}' will be restored to branch '{original_branch}' in " f"project '{project}'") self.temp_branches.append(BranchState(project, original_branch, name)) return name
def validate_content( self, selectors: List[str], exclusions: List[str], incremental: bool = False, exclude_personal: bool = False, ) -> Dict[str, Any]: with self.branch_manager: validator = ContentValidator(self.client, self.project, exclude_personal) logger.info("Building LookML project hierarchy for project " f"'{self.project}' @ {self.branch_manager.ref}") validator.build_project(selectors, exclusions) explore_count = validator.project.count_explores() print_header(f"Validating content based on {explore_count} " f"{'explore' if explore_count == 1 else 'explores'}" + (" [incremental mode] " if incremental else "")) results = validator.validate() if incremental and self.branch_manager.name != "master": logger.debug("Starting another content validation against master") self.branch_manager.commit_ref = None self.branch_manager.name = "master" with self.branch_manager: logger.debug("Building LookML project hierarchy for project " f"'{self.project}' @ {self.branch_manager.ref}") validator.build_project(selectors, exclusions) main_results = validator.validate() return self._incremental_results(main=main_results, additional=results) else: return results
def create_query(self, model: str, explore: str, dimensions: List[str], fields: List = []) -> Dict: """Creates a Looker async query for one or more specified dimensions. The query created is a SELECT query, selecting all dimensions specified for a certain model and explore. Looker builds the query using the `sql` field in the LookML for each dimension. If a Timeout exception is received, attempts to retry. """ # Using old-style string formatting so that strings are formatted lazily logger.debug( "Creating async query for %s/%s/%s", model, explore, "*" if len(dimensions) != 1 else dimensions[0], ) body = { "model": model, "view": explore, "fields": dimensions, "limit": 0, "filter_expression": "1=2", } params = {} if fields: params["fields"] = fields url = utils.compose_url(self.api_url, path=["queries"], params=params) response = self.post(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-create-query", title="Couldn't create query.", status=response.status_code, detail=(f"Failed to create query for {model}/{explore}/" f'{"*" if len(dimensions) > 1 else dimensions[0]}. ' "Please try again."), response=response, ) result = response.json() query_id = result["id"] logger.debug( "Query for %s/%s/%s created as query %d", model, explore, "*" if len(dimensions) != 1 else dimensions[0], query_id, ) return result
async def shutdown(self, signal, loop): logger.info("\n\n" + "Please wait, asking Looker to cancel any running queries") logger.debug("Cleaning up async tasks.") tasks = [ t for t in asyncio.all_tasks() if t is not asyncio.current_task() ] for task in tasks: task.cancel() await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
def cancel_query_task(self, query_task_id: str): """Cancels a query task. Args: query_task_id: ID for the query task to cancel. """ logger.debug(f"Cancelling query task: {query_task_id}") url = utils.compose_url(self.api_url, path=["running_queries", query_task_id]) self.delete(url=url, timeout=TIMEOUT_SEC)
def _fill_query_slots(self, queries: List[Query]) -> None: """Creates query tasks until all slots are used or all queries are running""" while queries and self.query_slots > 0: logger.debug( f"{self.query_slots} available query slots, creating query task" ) query = queries.pop(0) query_task_id = self.client.create_query_task(query.query_id) self.query_slots -= 1 query.query_task_id = query_task_id self._query_by_task_id[query_task_id] = query self._running_queries.append(query)