def update_branch(self, project: str, branch: str, ref: str = "origin/master"):
"""Updates a branch to the ref prodvided.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to update.
ref: The ref to update the branch from.
"""
logger.debug(
f"Updating branch '{branch}' on project '{project}' to ref '{ref}'"
)
body = {"name": branch, "ref": ref}
url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"])
response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-update-branch",
title="Couldn't update Git branch.",
status=response.status_code,
detail=(
f"Unable to update branch '{branch}' "
f"in project '{project}' using ref '{ref}'. "
"Please try again."
),
response=response,
)
def create_branch(self, project: str, branch: str, ref: str = "origin/master"):
"""Creates a branch in the given project.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to create.
ref: The ref to create the branch from.
"""
logger.debug(
f"Creating branch '{branch}' on project '{project}' with ref '{ref}'"
)
body = {"name": branch, "ref": ref}
url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"])
response = self.post(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-create-branch",
title="Couldn't create new Git branch.",
status=response.status_code,
detail=(
f"Unable to create branch '{branch}' "
f"in project '{project}' using ref '{ref}'. "
"Confirm the branch doesn't already exist and try again."
),
response=response,
)
def reset_to_remote(self, project: str) -> None:
"""Reset a project development branch to the revision of the project that is on the remote.
Args:
project: Name of the Looker project to use.
"""
logger.debug(f"Resetting branch to remote.")
url = utils.compose_url(
self.api_url, path=["projects", project, "reset_to_remote"]
)
response = self.post(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-reset-remote",
title="Couldn't checkout Git branch.",
status=response.status_code,
detail=(
f"Unable to reset local Git branch"
"to match remote. Please try again."
),
response=response,
)
def get_all_branches(self, project: str) -> List[str]:
"""Returns a list of git branches in the project repository.
Args:
project: Name of the Looker project to use.
"""
logger.debug(f"Getting all Git branches in project '{project}'")
url = utils.compose_url(
self.api_url, path=["projects", project, "git_branches"]
)
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-branches",
title="Couldn't get all Git branches.",
status=response.status_code,
detail=(
f"Unable to get all Git branches in project '{project}'. "
"Please try again."
),
response=response,
)
return [branch["name"] for branch in response.json()]
def delete_branch(self, project: str, branch: str):
"""Deletes a branch in the given project.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to delete.
"""
logger.debug(f"Deleting branch '{branch}' in project '{project}'")
url = utils.compose_url(
self.api_url, path=["projects", project, "git_branch", branch]
)
response = self.delete(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-delete-branch",
title="Couldn't delete Git branch.",
status=response.status_code,
detail=(
f"Unable to delete branch '{branch}' "
f"in project '{project}'. Please try again."
),
response=response,
)
def all_lookml_tests(self, project: str) -> List[JsonDict]:
"""Gets all LookML/data tests for a given project.
Args:
project: Name of the Looker project to use
Returns:
List[JsonDict]: JSON response containing all LookML/data tests
"""
logger.debug(f"Getting LookML tests for project {project}")
url = utils.compose_url(self.api_url,
path=["projects", project, "lookml_tests"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-data-tests",
title="Couldn't retrieve all data tests.",
status=response.status_code,
detail=(f"Unable to retrieve all data tests for "
f"project '{project}'. Please try again."),
response=response,
)
return response.json()
async def get_query_task_multi_results(
self, session: aiohttp.ClientSession, query_task_ids: List[str]
) -> JsonDict:
"""Returns query task results.
If a ClientError or TimeoutError is received, attempts to retry.
Args:
query_task_ids: IDs for the query tasks running asynchronously.
Returns:
List[JsonDict]: JSON response from the query task.
"""
# Using old-style string formatting so that strings are formatted lazily
logger.debug(
"Attempting to get results for %d query tasks", len(query_task_ids)
)
url = utils.compose_url(self.api_url, path=["query_tasks", "multi_results"])
async with session.get(
url=url, params={"query_task_ids": ",".join(query_task_ids)}
) as response:
result = await response.json()
response.raise_for_status()
return result
def wrapper(*args, **kwargs):
try:
return function(*args, **kwargs)
except ValidationError as error:
sys.exit(error.exit_code)
except SpectaclesException as error:
logger.error(
f"{error}\n\n"
+ printer.dim(
"For support, please create an issue at "
"https://github.com/spectacles-ci/spectacles/issues"
)
+ "\n"
)
sys.exit(error.exit_code)
except Exception as error:
logger.debug(error, exc_info=True)
logger.error(
f'Encountered unexpected {error.__class__.__name__}: "{error}"\n'
f"Full error traceback logged to {LOG_FILEPATH}\n\n"
+ printer.dim(
"For support, please create an issue at "
"https://github.com/spectacles-ci/spectacles/issues"
)
+ "\n"
)
sys.exit(1)
def get_lookml_models(self, fields: List = []) -> List[JsonDict]:
"""Gets all models and explores from the LookmlModel endpoint.
Returns:
List[JsonDict]: JSON response containing LookML models and explores.
"""
logger.debug(f"Getting all models and explores from {self.base_url}")
params = {}
if fields:
params["fields"] = fields
url = utils.compose_url(self.api_url,
path=["lookml_models"],
params=params)
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-lookml",
title="Couldn't retrieve models and explores.",
status=response.status_code,
detail="Unable to retrieve LookML details. Please try again.",
response=response,
)
return response.json()
def _get_query_results(self,
query_task_ids: List[str]) -> List[QueryResult]:
"""Returns ID, status, and error message for all query tasks"""
query_results = []
results = self.client.get_query_task_multi_results(query_task_ids)
for query_task_id, result in results.items():
status = result["status"]
if status not in ("complete", "error", "running", "added",
"expired"):
raise SpectaclesException(
name="unexpected-query-result-status",
title="Encountered an unexpected query result status.",
detail=(f"Query result status '{status}' was returned "
"by the Looker API."),
)
logger.debug(f"Query task {query_task_id} status is: {status}")
query_result = QueryResult(query_task_id, status)
if status == "error":
try:
error_details = self._extract_error_details(result)
except (KeyError, TypeError, IndexError) as error:
logger.debug(
f"Exiting because of unexpected query result format: {result}"
)
raise SpectaclesException(
name="unexpected-query-result-format",
title="Encountered an unexpected query result format.",
detail=
f"Unable to extract error details. The unexpected result has been logged.",
) from error
else:
query_result.error = error_details
query_results.append(query_result)
return query_results
def hard_reset_branch(self, project: str, branch: str, ref: str):
"""Hard resets a branch to the ref prodvided.
DANGER: hard reset will be force pushed to the remote. Unsaved changes and
commits may be permanently lost.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to update.
ref: The ref to update the branch from.
"""
logger.debug(
f"Hard resetting branch '{branch}' on project '{project}' to ref '{ref}'"
)
body = {"name": branch, "ref": ref}
url = utils.compose_url(self.api_url,
path=["projects", project, "git_branch"])
response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-update-branch",
title="Couldn't update Git branch.",
status=response.status_code,
detail=(f"Unable to update branch '{branch}' "
f"in project '{project}' using ref '{ref}'. "
"Please try again."),
response=response,
)
def update_workspace(self, project: str, workspace: str) -> None:
"""Updates the session workspace.
Args:
project: Name of the Looker project to use.
workspace: The workspace to switch to, either 'production' or 'dev'
"""
logger.debug(f"Updating session to use the {workspace} workspace")
url = utils.compose_url(self.api_url, path=["session"])
body = {"workspace_id": workspace}
response = self.patch(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-update-workspace",
title="Couldn't update the session's workspace.",
status=response.status_code,
detail=(
f"Unable to update workspace to '{workspace}'. "
"If you have any unsaved work on the branch "
"checked out by the user whose API credentials "
"Spectacles is using, please save it and try again."
),
response=response,
)
def get_manifest(self, project: str) -> JsonDict:
"""Gets all the dependent LookML projects defined in the manifest file.
Args:
project: Name of the Looker project to use.
Returns:
List[JsonDict]: JSON response containing all dependent projects
"""
logger.debug("Getting manifest details")
url = utils.compose_url(self.api_url,
path=["projects", project, "manifest"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-manifest",
title="Couldn't retrieve project manifest.",
status=response.status_code,
detail=
(f"Failed to retrieve manifest for project '{project}'. "
"Make sure you have a 'manifest.lkml' file in your project, "
"then try again."),
response=response,
)
manifest = response.json()
return manifest
def get_active_branch(self, project: str) -> JsonDict:
"""Gets the active branch for the user in the given project.
Args:
project: Name of the Looker project to use.
Returns:
str: Name of the active branch
"""
logger.debug(f"Getting active branch for project '{project}'")
url = utils.compose_url(self.api_url,
path=["projects", project, "git_branch"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-active-branch",
title="Couldn't determine active Git branch.",
status=response.status_code,
detail=(
f"Unable to get active branch for project '{project}'. "
"Please check that the project exists and try again."),
response=response,
)
branch_name = response.json()["name"]
logger.debug(f"The active branch is '{branch_name}'")
return response.json()
def checkout_branch(self, project: str, branch: str) -> None:
"""Checks out a new git branch. Only works in dev workspace.
Args:
project: Name of the Looker project to use.
branch: Name of the Git branch to check out.
"""
logger.debug(f"Setting project '{project}' branch to '{branch}'")
url = utils.compose_url(self.api_url,
path=["projects", project, "git_branch"])
body = {"name": branch}
response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-checkout-branch",
title="Couldn't checkout Git branch.",
status=response.status_code,
detail=(
f"Unable to checkout Git branch '{branch}'. "
"If you have uncommitted changes on the current branch, "
"please commit or revert them, then try again."),
response=response,
)
def get_looker_release_version(self) -> str:
"""Gets the version number of connected Looker instance.
Returns:
str: Looker instance release version number (e.g. 6.22.12)
"""
logger.debug("Checking Looker instance release version")
url = utils.compose_url(self.api_url, path=["versions"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-version",
title="Couldn't get Looker's release version.",
status=response.status_code,
detail=
("Unable to get the release version of your Looker instance. "
"Please try again."),
response=response,
)
return response.json()["looker_release_version"]
def request(self, method: str, url: str, *args,
**kwargs) -> requests.Response:
if self.access_token and self.access_token.expired:
logger.debug(
"Looker API access token has expired, requesting a new one")
self.authenticate()
return self.session.request(method, url, *args, **kwargs)
def authenticate(self, client_id: str, client_secret: str,
api_version: float) -> None:
"""Logs in to Looker's API using a client ID/secret pair and an API version.
Args:
client_id: Looker API client ID.
client_secret: Looker API client secret.
api_version: Desired API version to use for requests.
"""
logger.debug("Authenticating Looker API credentials")
url = utils.compose_url(self.api_url, path=["login"])
body = {"client_id": client_id, "client_secret": client_secret}
response = self.session.post(url=url, data=body)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
details = utils.details_from_http_error(response)
raise ApiConnectionError(
f"Failed to authenticate to {url}\n"
f"Attempted authentication with client ID {client_id}\n"
f"Looker API error encountered: {error}\n" +
"Message received from Looker's API: "
f'"{details}"')
access_token = response.json()["access_token"]
self.session.headers = {"Authorization": f"token {access_token}"}
logger.info(f"Connected using Looker API {api_version}")
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]:
"""Gets all dimensions for an explore from the LookmlModel endpoint.
Args:
model: Name of LookML model to query.
explore: Name of LookML explore to query.
Returns:
List[str]: Names of all the dimensions in the specified explore. Dimension
names are returned in the format 'explore_name.dimension_name'.
"""
logger.debug(f"Getting all dimensions from explore {explore}")
params = {"fields": ["fields"]}
url = utils.compose_url(
self.api_url,
path=["lookml_models", model, "explores", explore],
params=params,
)
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-dimension-lookml",
title="Couldn't retrieve dimensions.",
status=response.status_code,
detail=("Unable to retrieve dimension LookML details "
f"for explore '{model}/{explore}'. Please try again."),
response=response,
)
return response.json()["fields"]["dimensions"]
def get_query_task_multi_results(self, query_task_ids: List[str]) -> JsonDict:
"""Returns query task results.
If a ClientError or TimeoutError is received, attempts to retry.
Args:
query_task_ids: IDs for the query tasks running asynchronously.
Returns:
List[JsonDict]: JSON response from the query task.
"""
# Using old-style string formatting so that strings are formatted lazily
logger.debug(
"Attempting to get results for %d query tasks", len(query_task_ids)
)
url = utils.compose_url(self.api_url, path=["query_tasks", "multi_results"])
response = self.session.get(
url=url, params={"query_task_ids": ",".join(query_task_ids)}
)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
details = utils.details_from_http_error(response)
raise ApiConnectionError(
f"Looker API error encountered: {error}\n"
+ "Message received from Looker's API: "
f'"{details}"'
)
return response.json()
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]:
"""Gets all dimensions for an explore from the LookmlModel endpoint.
Args:
model: Name of LookML model to query.
explore: Name of LookML explore to query.
Returns:
List[str]: Names of all the dimensions in the specified explore. Dimension
names are returned in the format 'explore_name.dimension_name'.
"""
logger.debug(f"Getting all dimensions from explore {explore}")
url = utils.compose_url(
self.api_url, path=["lookml_models", model, "explores", explore]
)
response = self.session.get(url=url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
details = utils.details_from_http_error(response)
raise ApiConnectionError(
f'Unable to get dimensions for explore "{explore}".\n'
f"Looker API error encountered: {error}\n"
+ "Message received from Looker's API: "
f'"{details}"'
)
return response.json()["fields"]["dimensions"]
async def create_query_task(
self, session: aiohttp.ClientSession, query_id: int
) -> str:
"""Runs a previously created query asynchronously and returns the query task ID.
If a ClientError or TimeoutError is received, attempts to retry.
Args:
session: Existing asychronous HTTP session.
query_id: ID of a previously created query to run.
Returns:
str: ID for the query task, used to check on the status of the query, which
is being run asynchronously.
"""
# Using old-style string formatting so that strings are formatted lazily
logger.debug("Starting query %d", query_id)
body = {"query_id": query_id, "result_format": "json_detail"}
url = utils.compose_url(self.api_url, path=["query_tasks"])
async with session.post(
url=url, json=body, params={"cache": "false"}
) as response:
result = await response.json()
response.raise_for_status()
query_task_id = result["id"]
logger.debug("Query %d is running under query task %s", query_id, query_task_id)
return query_task_id
def run_lookml_test(self, project: str, model: str = None) -> List[JsonDict]:
"""Runs all LookML/data tests for a given project and model (optional)
This command only runs tests in production, as the Looker API doesn't currently
allow us to run data tests on a specific branch.
Args:
project: Name of the Looker project to use
model: Optional name of the LookML model to restrict testing to
Returns:
List[JsonDict]: JSON response containing any LookML/data test errors
"""
logger.debug(f"Running LookML tests for project {project}")
url = utils.compose_url(
self.api_url, path=["projects", project, "lookml_tests", "run"]
)
if model is not None:
response = self.session.get(url=url, params={"model": model})
else:
response = self.session.get(url=url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
raise ApiConnectionError(
f"Failed to run data tests for project {project}\n"
f'Error raised: "{error}"'
)
return response.json()
def all_lookml_tests(self, project: str) -> List[JsonDict]:
"""Gets all LookML/data tests for a given project.
Args:
project: Name of the Looker project to use
Returns:
List[JsonDict]: JSON response containing all LookML/data tests
"""
logger.debug(f"Getting LookML tests for project {project}")
url = utils.compose_url(
self.api_url, path=["projects", project, "lookml_tests"]
)
response = self.session.get(url=url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
raise ApiConnectionError(
f"Failed to retrieve data tests for project {project}\n"
f'Error raised: "{error}"'
)
return response.json()
def setup_temp_branch(self, project: str, original_branch: str) -> str:
name = "tmp_spectacles_" + time_hash()
logger.debug(
f"Branch '{name}' will be restored to branch '{original_branch}' in "
f"project '{project}'")
self.temp_branches.append(BranchState(project, original_branch, name))
return name
def validate_content(
self,
selectors: List[str],
exclusions: List[str],
incremental: bool = False,
exclude_personal: bool = False,
) -> Dict[str, Any]:
with self.branch_manager:
validator = ContentValidator(self.client, self.project,
exclude_personal)
logger.info("Building LookML project hierarchy for project "
f"'{self.project}' @ {self.branch_manager.ref}")
validator.build_project(selectors, exclusions)
explore_count = validator.project.count_explores()
print_header(f"Validating content based on {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'}" +
(" [incremental mode] " if incremental else ""))
results = validator.validate()
if incremental and self.branch_manager.name != "master":
logger.debug("Starting another content validation against master")
self.branch_manager.commit_ref = None
self.branch_manager.name = "master"
with self.branch_manager:
logger.debug("Building LookML project hierarchy for project "
f"'{self.project}' @ {self.branch_manager.ref}")
validator.build_project(selectors, exclusions)
main_results = validator.validate()
return self._incremental_results(main=main_results,
additional=results)
else:
return results
def create_query(self,
model: str,
explore: str,
dimensions: List[str],
fields: List = []) -> Dict:
"""Creates a Looker async query for one or more specified dimensions.
The query created is a SELECT query, selecting all dimensions specified for a
certain model and explore. Looker builds the query using the `sql` field in the
LookML for each dimension.
If a Timeout exception is received, attempts to retry.
"""
# Using old-style string formatting so that strings are formatted lazily
logger.debug(
"Creating async query for %s/%s/%s",
model,
explore,
"*" if len(dimensions) != 1 else dimensions[0],
)
body = {
"model": model,
"view": explore,
"fields": dimensions,
"limit": 0,
"filter_expression": "1=2",
}
params = {}
if fields:
params["fields"] = fields
url = utils.compose_url(self.api_url, path=["queries"], params=params)
response = self.post(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-create-query",
title="Couldn't create query.",
status=response.status_code,
detail=(f"Failed to create query for {model}/{explore}/"
f'{"*" if len(dimensions) > 1 else dimensions[0]}. '
"Please try again."),
response=response,
)
result = response.json()
query_id = result["id"]
logger.debug(
"Query for %s/%s/%s created as query %d",
model,
explore,
"*" if len(dimensions) != 1 else dimensions[0],
query_id,
)
return result
async def shutdown(self, signal, loop):
logger.info("\n\n" +
"Please wait, asking Looker to cancel any running queries")
logger.debug("Cleaning up async tasks.")
tasks = [
t for t in asyncio.all_tasks() if t is not asyncio.current_task()
]
for task in tasks:
task.cancel()
await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
def cancel_query_task(self, query_task_id: str):
"""Cancels a query task.
Args:
query_task_id: ID for the query task to cancel.
"""
logger.debug(f"Cancelling query task: {query_task_id}")
url = utils.compose_url(self.api_url, path=["running_queries", query_task_id])
self.delete(url=url, timeout=TIMEOUT_SEC)
def _fill_query_slots(self, queries: List[Query]) -> None:
"""Creates query tasks until all slots are used or all queries are running"""
while queries and self.query_slots > 0:
logger.debug(
f"{self.query_slots} available query slots, creating query task"
)
query = queries.pop(0)
query_task_id = self.client.create_query_task(query.query_id)
self.query_slots -= 1
query.query_task_id = query_task_id
self._query_by_task_id[query_task_id] = query
self._running_queries.append(query)
