def update_session(self, project: str, branch: str) -> None: """Switches to a development mode session and checks out the desired branch. Args: project: Name of the Looker project to use. branch: Name of the Git branch to check out. """ if branch == "master": logger.debug("Updating session to use production workspace") url = utils.compose_url(self.api_url, path=["session"]) body = {"workspace_id": "production"} response = self.session.patch(url=url, json=body) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f"Unable to update session to production workspace.\n" f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"' ) else: logger.debug("Updating session to use development workspace") url = utils.compose_url(self.api_url, path=["session"]) body = {"workspace_id": "dev"} response = self.session.patch(url=url, json=body) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f"Unable to update session to development workspace.\n" f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"' ) logger.debug(f"Setting Git branch to {branch}") url = utils.compose_url( self.api_url, path=["projects", project, "git_branch"] ) body = {"name": branch} response = self.session.put(url=url, json=body) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f"Unable to checkout Git branch {branch}. " "If you have uncommitted changes on the current branch, " "please commit or revert them, then try again.\n\n" f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"' ) logger.info(f"Checked out branch {branch}")
async def get_query_task_multi_results( self, session: aiohttp.ClientSession, query_task_ids: List[str] ) -> JsonDict: """Returns query task results. If a ClientError or TimeoutError is received, attempts to retry. Args: query_task_ids: IDs for the query tasks running asynchronously. Returns: List[JsonDict]: JSON response from the query task. """ # Using old-style string formatting so that strings are formatted lazily logger.debug( "Attempting to get results for %d query tasks", len(query_task_ids) ) url = utils.compose_url(self.api_url, path=["query_tasks", "multi_results"]) async with session.get( url=url, params={"query_task_ids": ",".join(query_task_ids)} ) as response: result = await response.json() response.raise_for_status() return result
def get_query_task_multi_results(self, query_task_ids: List[str]) -> JsonDict: """Returns query task results. If a ClientError or TimeoutError is received, attempts to retry. Args: query_task_ids: IDs for the query tasks running asynchronously. Returns: List[JsonDict]: JSON response from the query task. """ # Using old-style string formatting so that strings are formatted lazily logger.debug( "Attempting to get results for %d query tasks", len(query_task_ids) ) url = utils.compose_url(self.api_url, path=["query_tasks", "multi_results"]) response = self.session.get( url=url, params={"query_task_ids": ",".join(query_task_ids)} ) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"' ) return response.json()
def update_branch(self, project: str, branch: str, ref: str = "origin/master"): """Updates a branch to the ref prodvided. Args: project: Name of the Looker project to use. branch: Name of the branch to update. ref: The ref to update the branch from. """ logger.debug( f"Updating branch '{branch}' on project '{project}' to ref '{ref}'" ) body = {"name": branch, "ref": ref} url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.put(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-update-branch", title="Couldn't update Git branch.", status=response.status_code, detail=( f"Unable to update branch '{branch}' " f"in project '{project}' using ref '{ref}'. " "Please try again." ), response=response, )
def create_branch(self, project: str, branch: str, ref: str = "origin/master"): """Creates a branch in the given project. Args: project: Name of the Looker project to use. branch: Name of the branch to create. ref: The ref to create the branch from. """ logger.debug( f"Creating branch '{branch}' on project '{project}' with ref '{ref}'" ) body = {"name": branch, "ref": ref} url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.post(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-create-branch", title="Couldn't create new Git branch.", status=response.status_code, detail=( f"Unable to create branch '{branch}' " f"in project '{project}' using ref '{ref}'. " "Confirm the branch doesn't already exist and try again." ), response=response, )
def reset_to_remote(self, project: str) -> None: """Reset a project development branch to the revision of the project that is on the remote. Args: project: Name of the Looker project to use. """ logger.debug(f"Resetting branch to remote.") url = utils.compose_url( self.api_url, path=["projects", project, "reset_to_remote"] ) response = self.post(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-reset-remote", title="Couldn't checkout Git branch.", status=response.status_code, detail=( f"Unable to reset local Git branch" "to match remote. Please try again." ), response=response, )
def get_all_branches(self, project: str) -> List[str]: """Returns a list of git branches in the project repository. Args: project: Name of the Looker project to use. """ logger.debug(f"Getting all Git branches in project '{project}'") url = utils.compose_url( self.api_url, path=["projects", project, "git_branches"] ) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-branches", title="Couldn't get all Git branches.", status=response.status_code, detail=( f"Unable to get all Git branches in project '{project}'. " "Please try again." ), response=response, ) return [branch["name"] for branch in response.json()]
def delete_branch(self, project: str, branch: str): """Deletes a branch in the given project. Args: project: Name of the Looker project to use. branch: Name of the branch to delete. """ logger.debug(f"Deleting branch '{branch}' in project '{project}'") url = utils.compose_url( self.api_url, path=["projects", project, "git_branch", branch] ) response = self.delete(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-delete-branch", title="Couldn't delete Git branch.", status=response.status_code, detail=( f"Unable to delete branch '{branch}' " f"in project '{project}'. Please try again." ), response=response, )
def all_lookml_tests(self, project: str) -> List[JsonDict]: """Gets all LookML/data tests for a given project. Args: project: Name of the Looker project to use Returns: List[JsonDict]: JSON response containing all LookML/data tests """ logger.debug(f"Getting LookML tests for project {project}") url = utils.compose_url(self.api_url, path=["projects", project, "lookml_tests"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-data-tests", title="Couldn't retrieve all data tests.", status=response.status_code, detail=(f"Unable to retrieve all data tests for " f"project '{project}'. Please try again."), response=response, ) return response.json()
def hard_reset_branch(self, project: str, branch: str, ref: str): """Hard resets a branch to the ref prodvided. DANGER: hard reset will be force pushed to the remote. Unsaved changes and commits may be permanently lost. Args: project: Name of the Looker project to use. branch: Name of the branch to update. ref: The ref to update the branch from. """ logger.debug( f"Hard resetting branch '{branch}' on project '{project}' to ref '{ref}'" ) body = {"name": branch, "ref": ref} url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.put(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-update-branch", title="Couldn't update Git branch.", status=response.status_code, detail=(f"Unable to update branch '{branch}' " f"in project '{project}' using ref '{ref}'. " "Please try again."), response=response, )
def authenticate(self, client_id: str, client_secret: str, api_version: float) -> None: """Logs in to Looker's API using a client ID/secret pair and an API version. Args: client_id: Looker API client ID. client_secret: Looker API client secret. api_version: Desired API version to use for requests. """ logger.debug("Authenticating Looker API credentials") url = utils.compose_url(self.api_url, path=["login"]) body = {"client_id": client_id, "client_secret": client_secret} response = self.session.post(url=url, data=body) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f"Failed to authenticate to {url}\n" f"Attempted authentication with client ID {client_id}\n" f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"') access_token = response.json()["access_token"] self.session.headers = {"Authorization": f"token {access_token}"} logger.info(f"Connected using Looker API {api_version}")
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]: """Gets all dimensions for an explore from the LookmlModel endpoint. Args: model: Name of LookML model to query. explore: Name of LookML explore to query. Returns: List[str]: Names of all the dimensions in the specified explore. Dimension names are returned in the format 'explore_name.dimension_name'. """ logger.debug(f"Getting all dimensions from explore {explore}") url = utils.compose_url( self.api_url, path=["lookml_models", model, "explores", explore] ) response = self.session.get(url=url) try: response.raise_for_status() except requests.exceptions.HTTPError as error: details = utils.details_from_http_error(response) raise ApiConnectionError( f'Unable to get dimensions for explore "{explore}".\n' f"Looker API error encountered: {error}\n" + "Message received from Looker's API: " f'"{details}"' ) return response.json()["fields"]["dimensions"]
def run_lookml_test(self, project: str, model: str = None) -> List[JsonDict]: """Runs all LookML/data tests for a given project and model (optional) This command only runs tests in production, as the Looker API doesn't currently allow us to run data tests on a specific branch. Args: project: Name of the Looker project to use model: Optional name of the LookML model to restrict testing to Returns: List[JsonDict]: JSON response containing any LookML/data test errors """ logger.debug(f"Running LookML tests for project {project}") url = utils.compose_url( self.api_url, path=["projects", project, "lookml_tests", "run"] ) if model is not None: response = self.session.get(url=url, params={"model": model}) else: response = self.session.get(url=url) try: response.raise_for_status() except requests.exceptions.HTTPError as error: raise ApiConnectionError( f"Failed to run data tests for project {project}\n" f'Error raised: "{error}"' ) return response.json()
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]: """Gets all dimensions for an explore from the LookmlModel endpoint. Args: model: Name of LookML model to query. explore: Name of LookML explore to query. Returns: List[str]: Names of all the dimensions in the specified explore. Dimension names are returned in the format 'explore_name.dimension_name'. """ logger.debug(f"Getting all dimensions from explore {explore}") params = {"fields": ["fields"]} url = utils.compose_url( self.api_url, path=["lookml_models", model, "explores", explore], params=params, ) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-dimension-lookml", title="Couldn't retrieve dimensions.", status=response.status_code, detail=("Unable to retrieve dimension LookML details " f"for explore '{model}/{explore}'. Please try again."), response=response, ) return response.json()["fields"]["dimensions"]
def get_looker_release_version(self) -> str: """Gets the version number of connected Looker instance. Returns: str: Looker instance release version number (e.g. 6.22.12) """ logger.debug("Checking Looker instance release version") url = utils.compose_url(self.api_url, path=["versions"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-version", title="Couldn't get Looker's release version.", status=response.status_code, detail= ("Unable to get the release version of your Looker instance. " "Please try again."), response=response, ) return response.json()["looker_release_version"]
async def create_query_task( self, session: aiohttp.ClientSession, query_id: int ) -> str: """Runs a previously created query asynchronously and returns the query task ID. If a ClientError or TimeoutError is received, attempts to retry. Args: session: Existing asychronous HTTP session. query_id: ID of a previously created query to run. Returns: str: ID for the query task, used to check on the status of the query, which is being run asynchronously. """ # Using old-style string formatting so that strings are formatted lazily logger.debug("Starting query %d", query_id) body = {"query_id": query_id, "result_format": "json_detail"} url = utils.compose_url(self.api_url, path=["query_tasks"]) async with session.post( url=url, json=body, params={"cache": "false"} ) as response: result = await response.json() response.raise_for_status() query_task_id = result["id"] logger.debug("Query %d is running under query task %s", query_id, query_task_id) return query_task_id
def get_lookml_models(self, fields: List = []) -> List[JsonDict]: """Gets all models and explores from the LookmlModel endpoint. Returns: List[JsonDict]: JSON response containing LookML models and explores. """ logger.debug(f"Getting all models and explores from {self.base_url}") params = {} if fields: params["fields"] = fields url = utils.compose_url(self.api_url, path=["lookml_models"], params=params) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-lookml", title="Couldn't retrieve models and explores.", status=response.status_code, detail="Unable to retrieve LookML details. Please try again.", response=response, ) return response.json()
def test_compose_url_multiple_path_components_and_one_field_params(): url = utils.compose_url( TEST_BASE_URL, ["api", "3.0", "login", "42", "auth", "27"], {"fields": ["joins"]}, ) assert url == "https://test.looker.com/api/3.0/login/42/auth/27?fields=joins"
def update_workspace(self, project: str, workspace: str) -> None: """Updates the session workspace. Args: project: Name of the Looker project to use. workspace: The workspace to switch to, either 'production' or 'dev' """ logger.debug(f"Updating session to use the {workspace} workspace") url = utils.compose_url(self.api_url, path=["session"]) body = {"workspace_id": workspace} response = self.patch(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-update-workspace", title="Couldn't update the session's workspace.", status=response.status_code, detail=( f"Unable to update workspace to '{workspace}'. " "If you have any unsaved work on the branch " "checked out by the user whose API credentials " "Spectacles is using, please save it and try again." ), response=response, )
def get_active_branch(self, project: str) -> JsonDict: """Gets the active branch for the user in the given project. Args: project: Name of the Looker project to use. Returns: str: Name of the active branch """ logger.debug(f"Getting active branch for project '{project}'") url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-active-branch", title="Couldn't determine active Git branch.", status=response.status_code, detail=( f"Unable to get active branch for project '{project}'. " "Please check that the project exists and try again."), response=response, ) branch_name = response.json()["name"] logger.debug(f"The active branch is '{branch_name}'") return response.json()
def get_manifest(self, project: str) -> JsonDict: """Gets all the dependent LookML projects defined in the manifest file. Args: project: Name of the Looker project to use. Returns: List[JsonDict]: JSON response containing all dependent projects """ logger.debug("Getting manifest details") url = utils.compose_url(self.api_url, path=["projects", project, "manifest"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-manifest", title="Couldn't retrieve project manifest.", status=response.status_code, detail= (f"Failed to retrieve manifest for project '{project}'. " "Make sure you have a 'manifest.lkml' file in your project, " "then try again."), response=response, ) manifest = response.json() return manifest
def checkout_branch(self, project: str, branch: str) -> None: """Checks out a new git branch. Only works in dev workspace. Args: project: Name of the Looker project to use. branch: Name of the Git branch to check out. """ logger.debug(f"Setting project '{project}' branch to '{branch}'") url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"]) body = {"name": branch} response = self.put(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-checkout-branch", title="Couldn't checkout Git branch.", status=response.status_code, detail=( f"Unable to checkout Git branch '{branch}'. " "If you have uncommitted changes on the current branch, " "please commit or revert them, then try again."), response=response, )
def all_lookml_tests(self, project: str) -> List[JsonDict]: """Gets all LookML/data tests for a given project. Args: project: Name of the Looker project to use Returns: List[JsonDict]: JSON response containing all LookML/data tests """ logger.debug(f"Getting LookML tests for project {project}") url = utils.compose_url( self.api_url, path=["projects", project, "lookml_tests"] ) response = self.session.get(url=url) try: response.raise_for_status() except requests.exceptions.HTTPError as error: raise ApiConnectionError( f"Failed to retrieve data tests for project {project}\n" f'Error raised: "{error}"' ) return response.json()
def create_query(self, model: str, explore: str, dimensions: List[str], fields: List = []) -> Dict: """Creates a Looker async query for one or more specified dimensions. The query created is a SELECT query, selecting all dimensions specified for a certain model and explore. Looker builds the query using the `sql` field in the LookML for each dimension. If a Timeout exception is received, attempts to retry. """ # Using old-style string formatting so that strings are formatted lazily logger.debug( "Creating async query for %s/%s/%s", model, explore, "*" if len(dimensions) != 1 else dimensions[0], ) body = { "model": model, "view": explore, "fields": dimensions, "limit": 0, "filter_expression": "1=2", } params = {} if fields: params["fields"] = fields url = utils.compose_url(self.api_url, path=["queries"], params=params) response = self.post(url=url, json=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-create-query", title="Couldn't create query.", status=response.status_code, detail=(f"Failed to create query for {model}/{explore}/" f'{"*" if len(dimensions) > 1 else dimensions[0]}. ' "Please try again."), response=response, ) result = response.json() query_id = result["id"] logger.debug( "Query for %s/%s/%s created as query %d", model, explore, "*" if len(dimensions) != 1 else dimensions[0], query_id, ) return result
def cancel_query_task(self, query_task_id: str): """Cancels a query task. Args: query_task_id: ID for the query task to cancel. """ logger.debug(f"Cancelling query task: {query_task_id}") url = utils.compose_url(self.api_url, path=["running_queries", query_task_id]) self.delete(url=url, timeout=TIMEOUT_SEC)
async def create_query( self, session: aiohttp.ClientSession, model: str, explore: str, dimensions: List[str], ) -> int: """Creates a Looker async query for one or more specified dimensions. The query created is a SELECT query, selecting all dimensions specified for a certain model and explore. Looker builds the query using the `sql` field in the LookML for each dimension. If a ClientError or TimeoutError is received, attempts to retry. Args: session: Existing asychronous HTTP session. model: Name of LookML model to query. explore: Name of LookML explore to query. dimensions: Names of the LookML dimensions in the specified explore to query. Returns: int: ID for the created query. """ # Using old-style string formatting so that strings are formatted lazily logger.debug( "Creating async query for %s/%s/%s", model, explore, "*" if len(dimensions) > 1 else dimensions[0], ) body = { "model": model, "view": explore, "fields": dimensions, "limit": 0, "filter_expression": "1=2", } url = utils.compose_url(self.api_url, path=["queries"]) async with session.post(url=url, json=body) as response: result = await response.json() response.raise_for_status() query_id = result["id"] logger.debug( "Query for %s/%s/%s created as query %d", model, explore, "*" if len(dimensions) > 1 else dimensions[0], query_id, ) return query_id
async def cancel_query_task( self, session: aiohttp.ClientSession, query_task_id: str ): """ Cancels a query task. Args: query_task_id: ID for the query task to cancel. """ logger.debug(f"Cancelling query task: {query_task_id}") url = utils.compose_url(self.api_url, path=["running_queries", query_task_id]) async with session.delete(url=url) as response: await response.read()
def run_lookml_test(self, project: str, model: str = None, test: str = None) -> List[JsonDict]: """Runs all LookML/data tests for a given project and model (optional) This command only runs tests in production, as the Looker API doesn't currently allow us to run data tests on a specific branch. Args: project: Name of the Looker project to use model: Optional name of the LookML model to restrict testing to Returns: List[JsonDict]: JSON response containing any LookML/data test errors """ if model is None and test is None: logger.debug(f"Running all LookML tests for project '{project}'") elif model is None and test is not None: logger.debug(f"Running LookML test '{test}'") elif model is not None and test is None: logger.debug(f"Running all LookML tests for model '{model}'") elif model is not None and test is not None: logger.debug(f"Running LookML test '{test}' in model '{model}'") url = utils.compose_url( self.api_url, path=["projects", project, "lookml_tests", "run"]) params = {} if model is not None: params["model"] = model if test is not None: params["test"] = test response = self.session.get(url=url, params=params, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-run-data-test", title="Couldn't run data test.", status=response.status_code, detail=(f"Unable to run one or more data tests for " f"project '{project}'. Please try again."), response=response, ) return response.json()
def all_folders(self, project: str) -> List[JsonDict]: logger.debug("Getting information about all folders") url = utils.compose_url(self.api_url, path=["folders"]) response = self.get(url=url, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-get-folders", title="Couldn't obtain project folders.", status=response.status_code, detail=(f"Failed to get all folders for project '{project}'."), response=response, ) result = response.json() return result
def authenticate(self) -> None: """Logs in to Looker's API using a client ID/secret pair and an API version. Args: client_id: Looker API client ID. client_secret: Looker API client secret. api_version: Desired API version to use for requests. """ logger.debug("Authenticating Looker API credentials") url = utils.compose_url(self.api_url, path=["login"]) body = { "client_id": self.client_id, "client_secret": self.client_secret } self.session.auth = NullAuth() # This should not use `self.post` or it will create a recursive loop response = self.session.post(url=url, data=body, timeout=TIMEOUT_SEC) try: response.raise_for_status() except requests.exceptions.HTTPError: raise LookerApiError( name="unable-to-authenticate", title="Couldn't authenticate to the Looker API.", status=response.status_code, detail= (f"Unable to authenticate with client ID '{self.client_id}'. " "Check that your credentials are correct and try again."), response=response, ) result = response.json() if "expires_at" not in result: # Calculate the expiration time with a one-minute buffer result["expires_at"] = time.time() + result["expires_in"] - 60 self.access_token = AccessToken(**result) self.session.headers = { # type: ignore "Authorization": f"token {self.access_token}" } looker_version = self.get_looker_release_version() logger.info(f"Connected to Looker version {looker_version} " f"using Looker API {self.api_version}")