def get_all_branches(self, project: str) -> List[str]:
"""Returns a list of git branches in the project repository.
Args:
project: Name of the Looker project to use.
"""
logger.debug(f"Getting all Git branches in project '{project}'")
url = utils.compose_url(
self.api_url, path=["projects", project, "git_branches"]
)
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-branches",
title="Couldn't get all Git branches.",
status=response.status_code,
detail=(
f"Unable to get all Git branches in project '{project}'. "
"Please try again."
),
response=response,
)
return [branch["name"] for branch in response.json()]
def run_sql(
project,
branch,
explores,
base_url,
client_id,
client_secret,
port,
api_version,
mode,
remote_reset,
concurrency,
) -> None:
"""Runs and validates the SQL for each selected LookML dimension."""
runner = Runner(
base_url,
project,
branch,
client_id,
client_secret,
port,
api_version,
remote_reset,
)
errors = runner.validate_sql(explores, mode, concurrency)
if errors:
for error in sorted(errors, key=lambda x: x["path"]):
printer.print_sql_error(error)
logger.info("")
raise ValidationError
else:
logger.info("")
def _get_query_results(self,
query_task_ids: List[str]) -> List[QueryResult]:
"""Returns ID, status, and error message for all query tasks"""
query_results = []
results = self.client.get_query_task_multi_results(query_task_ids)
for query_task_id, result in results.items():
status = result["status"]
if status not in ("complete", "error", "running", "added",
"expired"):
raise SpectaclesException(
name="unexpected-query-result-status",
title="Encountered an unexpected query result status.",
detail=(f"Query result status '{status}' was returned "
"by the Looker API."),
)
logger.debug(f"Query task {query_task_id} status is: {status}")
query_result = QueryResult(query_task_id, status)
if status == "error":
try:
error_details = self._extract_error_details(result)
except (KeyError, TypeError, IndexError) as error:
logger.debug(
f"Exiting because of unexpected query result format: {result}"
)
raise SpectaclesException(
name="unexpected-query-result-format",
title="Encountered an unexpected query result format.",
detail=
f"Unable to extract error details. The unexpected result has been logged.",
) from error
else:
query_result.error = error_details
query_results.append(query_result)
return query_results
def run_lookml_test(self, project: str, model: str = None) -> List[JsonDict]:
"""Runs all LookML/data tests for a given project and model (optional)
This command only runs tests in production, as the Looker API doesn't currently
allow us to run data tests on a specific branch.
Args:
project: Name of the Looker project to use
model: Optional name of the LookML model to restrict testing to
Returns:
List[JsonDict]: JSON response containing any LookML/data test errors
"""
logger.debug(f"Running LookML tests for project {project}")
url = utils.compose_url(
self.api_url, path=["projects", project, "lookml_tests", "run"]
)
if model is not None:
response = self.session.get(url=url, params={"model": model})
else:
response = self.session.get(url=url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
raise ApiConnectionError(
f"Failed to run data tests for project {project}\n"
f'Error raised: "{error}"'
)
return response.json()
async def create_query_task(
self, session: aiohttp.ClientSession, query_id: int
) -> str:
"""Runs a previously created query asynchronously and returns the query task ID.
If a ClientError or TimeoutError is received, attempts to retry.
Args:
session: Existing asychronous HTTP session.
query_id: ID of a previously created query to run.
Returns:
str: ID for the query task, used to check on the status of the query, which
is being run asynchronously.
"""
# Using old-style string formatting so that strings are formatted lazily
logger.debug("Starting query %d", query_id)
body = {"query_id": query_id, "result_format": "json_detail"}
url = utils.compose_url(self.api_url, path=["query_tasks"])
async with session.post(
url=url, json=body, params={"cache": "false"}
) as response:
result = await response.json()
response.raise_for_status()
query_task_id = result["id"]
logger.debug("Query %d is running under query task %s", query_id, query_task_id)
return query_task_id
def validate(self, mode: str = "batch") -> List[SqlError]:
"""Queries selected explores and returns any errors.
Args:
batch: When true, runs one query per explore (using all dimensions). When
false, runs one query per dimension. Batch mode increases query speed
but can only return the first error encountered for each dimension.
Returns:
List[SqlError]: SqlErrors encountered while querying the explore.
"""
explore_count = self._count_explores()
printer.print_header(
f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode]")
errors = self._query(mode)
if mode == "hybrid" and self.project.errored:
errors = self._query(mode)
for model in sorted(self.project.models, key=lambda x: x.name):
for explore in sorted(model.explores, key=lambda x: x.name):
if explore.errored:
logger.info(
f"✗ {printer.red(model.name + '.' + explore.name)} failed"
)
else:
logger.info(
f"✓ {printer.green(model.name + '.' + explore.name)} passed"
)
return errors
def setup_temp_branch(self, project: str, original_branch: str) -> str:
name = "tmp_spectacles_" + time_hash()
logger.debug(
f"Branch '{name}' will be restored to branch '{original_branch}' in "
f"project '{project}'")
self.temp_branches.append(BranchState(project, original_branch, name))
return name
def request(self, method: str, url: str, *args,
**kwargs) -> requests.Response:
if self.access_token and self.access_token.expired:
logger.debug(
"Looker API access token has expired, requesting a new one")
self.authenticate()
return self.session.request(method, url, *args, **kwargs)
def get_looker_release_version(self) -> str:
"""Gets the version number of connected Looker instance.
Returns:
str: Looker instance release version number (e.g. 6.22.12)
"""
logger.debug("Checking Looker instance release version")
url = utils.compose_url(self.api_url, path=["versions"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-version",
title="Couldn't get Looker's release version.",
status=response.status_code,
detail=
("Unable to get the release version of your Looker instance. "
"Please try again."),
response=response,
)
return response.json()["looker_release_version"]
def delete_branch(self, project: str, branch: str):
"""Deletes a branch in the given project.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to delete.
"""
logger.debug(f"Deleting branch '{branch}' in project '{project}'")
url = utils.compose_url(
self.api_url, path=["projects", project, "git_branch", branch]
)
response = self.delete(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-delete-branch",
title="Couldn't delete Git branch.",
status=response.status_code,
detail=(
f"Unable to delete branch '{branch}' "
f"in project '{project}'. Please try again."
),
response=response,
)
def print_data_test_error(model: str, explore: str, test_name: str,
message: str, lookml_url: str) -> None:
path = f"{model}/{explore}/{test_name}"
print_header(red(path), LINE_WIDTH + COLOR_CODE_LENGTH)
wrapped = textwrap.fill(message, LINE_WIDTH)
logger.info(wrapped)
logger.info("\n" + f"LookML: {lookml_url}")
def update_branch(self, project: str, branch: str, ref: str = "origin/master"):
"""Updates a branch to the ref prodvided.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to update.
ref: The ref to update the branch from.
"""
logger.debug(
f"Updating branch '{branch}' on project '{project}' to ref '{ref}'"
)
body = {"name": branch, "ref": ref}
url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"])
response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-update-branch",
title="Couldn't update Git branch.",
status=response.status_code,
detail=(
f"Unable to update branch '{branch}' "
f"in project '{project}' using ref '{ref}'. "
"Please try again."
),
response=response,
)
def create_branch(self, project: str, branch: str, ref: str = "origin/master"):
"""Creates a branch in the given project.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to create.
ref: The ref to create the branch from.
"""
logger.debug(
f"Creating branch '{branch}' on project '{project}' with ref '{ref}'"
)
body = {"name": branch, "ref": ref}
url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"])
response = self.post(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-create-branch",
title="Couldn't create new Git branch.",
status=response.status_code,
detail=(
f"Unable to create branch '{branch}' "
f"in project '{project}' using ref '{ref}'. "
"Confirm the branch doesn't already exist and try again."
),
response=response,
)
def reset_to_remote(self, project: str) -> None:
"""Reset a project development branch to the revision of the project that is on the remote.
Args:
project: Name of the Looker project to use.
"""
logger.debug(f"Resetting branch to remote.")
url = utils.compose_url(
self.api_url, path=["projects", project, "reset_to_remote"]
)
response = self.post(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-reset-remote",
title="Couldn't checkout Git branch.",
status=response.status_code,
detail=(
f"Unable to reset local Git branch"
"to match remote. Please try again."
),
response=response,
)
def validate_looker_release_version(self, required_version: str) -> bool:
"""Checks that the current Looker version meets a specified minimum.
Args:
required_version: Minimum instance version number (e.g. 6.22.12)
Returns:
bool: True if the current Looker version >= the required version
"""
current_version = self.get_looker_release_version()
logger.info(f"Looker instance version is {current_version}")
def expand_version(version: str):
return [int(number) for number in version.split(".")]
current = expand_version(current_version)
required = expand_version(required_version)
# If version is provided in format 6.20 or 7, extend with .0(s)
# e.g. 6.20 would become 6.20.0, 7 would become 7.0.0
if len(current) < 3:
current.extend([0] * (3 - len(current)))
for current_num, required_num in zip(current, required):
if current_num < required_num:
return False
elif current_num > required_num:
return True
# Loop exits successfully if current version == required version
return True
def checkout_branch(self, project: str, branch: str) -> None:
"""Checks out a new git branch. Only works in dev workspace.
Args:
project: Name of the Looker project to use.
branch: Name of the Git branch to check out.
"""
logger.debug(f"Setting project '{project}' branch to '{branch}'")
url = utils.compose_url(self.api_url,
path=["projects", project, "git_branch"])
body = {"name": branch}
response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-checkout-branch",
title="Couldn't checkout Git branch.",
status=response.status_code,
detail=(
f"Unable to checkout Git branch '{branch}'. "
"If you have uncommitted changes on the current branch, "
"please commit or revert them, then try again."),
response=response,
)
def authenticate(self, client_id: str, client_secret: str,
api_version: float) -> None:
"""Logs in to Looker's API using a client ID/secret pair and an API version.
Args:
client_id: Looker API client ID.
client_secret: Looker API client secret.
api_version: Desired API version to use for requests.
"""
logger.debug("Authenticating Looker API credentials")
url = utils.compose_url(self.api_url, path=["login"])
body = {"client_id": client_id, "client_secret": client_secret}
response = self.session.post(url=url, data=body)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
details = utils.details_from_http_error(response)
raise ApiConnectionError(
f"Failed to authenticate to {url}\n"
f"Attempted authentication with client ID {client_id}\n"
f"Looker API error encountered: {error}\n" +
"Message received from Looker's API: "
f'"{details}"')
access_token = response.json()["access_token"]
self.session.headers = {"Authorization": f"token {access_token}"}
logger.info(f"Connected using Looker API {api_version}")
def get_manifest(self, project: str) -> JsonDict:
"""Gets all the dependent LookML projects defined in the manifest file.
Args:
project: Name of the Looker project to use.
Returns:
List[JsonDict]: JSON response containing all dependent projects
"""
logger.debug("Getting manifest details")
url = utils.compose_url(self.api_url,
path=["projects", project, "manifest"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-manifest",
title="Couldn't retrieve project manifest.",
status=response.status_code,
detail=
(f"Failed to retrieve manifest for project '{project}'. "
"Make sure you have a 'manifest.lkml' file in your project, "
"then try again."),
response=response,
)
manifest = response.json()
return manifest
def validate_sql(
self,
branch: Optional[str],
commit: Optional[str],
selectors: List[str],
exclusions: List[str],
mode: QueryMode = "batch",
concurrency: int = 10,
profile: bool = False,
runtime_threshold: int = 5,
) -> Dict[str, Any]:
with self.branch_manager(branch, commit):
validator = SqlValidator(self.client, self.project, concurrency,
runtime_threshold)
logger.info("Building LookML project hierarchy for project "
f"'{self.project}' @ {self.branch_manager.ref}")
validator.build_project(selectors,
exclusions,
build_dimensions=True)
explore_count = validator.project.count_explores()
print_header(f"Testing {explore_count} "
f"{'explore' if explore_count == 1 else 'explores'} "
f"[{mode} mode] "
f"[concurrency = {validator.query_slots}]")
results = validator.validate(mode, profile)
return results
def get_active_branch(self, project: str) -> JsonDict:
"""Gets the active branch for the user in the given project.
Args:
project: Name of the Looker project to use.
Returns:
str: Name of the active branch
"""
logger.debug(f"Getting active branch for project '{project}'")
url = utils.compose_url(self.api_url,
path=["projects", project, "git_branch"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-active-branch",
title="Couldn't determine active Git branch.",
status=response.status_code,
detail=(
f"Unable to get active branch for project '{project}'. "
"Please check that the project exists and try again."),
response=response,
)
branch_name = response.json()["name"]
logger.debug(f"The active branch is '{branch_name}'")
return response.json()
def all_lookml_tests(self, project: str) -> List[JsonDict]:
"""Gets all LookML/data tests for a given project.
Args:
project: Name of the Looker project to use
Returns:
List[JsonDict]: JSON response containing all LookML/data tests
"""
logger.debug(f"Getting LookML tests for project {project}")
url = utils.compose_url(
self.api_url, path=["projects", project, "lookml_tests"]
)
response = self.session.get(url=url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
raise ApiConnectionError(
f"Failed to retrieve data tests for project {project}\n"
f'Error raised: "{error}"'
)
return response.json()
def hard_reset_branch(self, project: str, branch: str, ref: str):
"""Hard resets a branch to the ref prodvided.
DANGER: hard reset will be force pushed to the remote. Unsaved changes and
commits may be permanently lost.
Args:
project: Name of the Looker project to use.
branch: Name of the branch to update.
ref: The ref to update the branch from.
"""
logger.debug(
f"Hard resetting branch '{branch}' on project '{project}' to ref '{ref}'"
)
body = {"name": branch, "ref": ref}
url = utils.compose_url(self.api_url,
path=["projects", project, "git_branch"])
response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-update-branch",
title="Couldn't update Git branch.",
status=response.status_code,
detail=(f"Unable to update branch '{branch}' "
f"in project '{project}' using ref '{ref}'. "
"Please try again."),
response=response,
)
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]:
"""Gets all dimensions for an explore from the LookmlModel endpoint.
Args:
model: Name of LookML model to query.
explore: Name of LookML explore to query.
Returns:
List[str]: Names of all the dimensions in the specified explore. Dimension
names are returned in the format 'explore_name.dimension_name'.
"""
logger.debug(f"Getting all dimensions from explore {explore}")
url = utils.compose_url(
self.api_url, path=["lookml_models", model, "explores", explore]
)
response = self.session.get(url=url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
details = utils.details_from_http_error(response)
raise ApiConnectionError(
f'Unable to get dimensions for explore "{explore}".\n'
f"Looker API error encountered: {error}\n"
+ "Message received from Looker's API: "
f'"{details}"'
)
return response.json()["fields"]["dimensions"]
def all_lookml_tests(self, project: str) -> List[JsonDict]:
"""Gets all LookML/data tests for a given project.
Args:
project: Name of the Looker project to use
Returns:
List[JsonDict]: JSON response containing all LookML/data tests
"""
logger.debug(f"Getting LookML tests for project {project}")
url = utils.compose_url(self.api_url,
path=["projects", project, "lookml_tests"])
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-data-tests",
title="Couldn't retrieve all data tests.",
status=response.status_code,
detail=(f"Unable to retrieve all data tests for "
f"project '{project}'. Please try again."),
response=response,
)
return response.json()
def get_query_task_multi_results(self, query_task_ids: List[str]) -> JsonDict:
"""Returns query task results.
If a ClientError or TimeoutError is received, attempts to retry.
Args:
query_task_ids: IDs for the query tasks running asynchronously.
Returns:
List[JsonDict]: JSON response from the query task.
"""
# Using old-style string formatting so that strings are formatted lazily
logger.debug(
"Attempting to get results for %d query tasks", len(query_task_ids)
)
url = utils.compose_url(self.api_url, path=["query_tasks", "multi_results"])
response = self.session.get(
url=url, params={"query_task_ids": ",".join(query_task_ids)}
)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as error:
details = utils.details_from_http_error(response)
raise ApiConnectionError(
f"Looker API error encountered: {error}\n"
+ "Message received from Looker's API: "
f'"{details}"'
)
return response.json()
def get_lookml_models(self, fields: List = []) -> List[JsonDict]:
"""Gets all models and explores from the LookmlModel endpoint.
Returns:
List[JsonDict]: JSON response containing LookML models and explores.
"""
logger.debug(f"Getting all models and explores from {self.base_url}")
params = {}
if fields:
params["fields"] = fields
url = utils.compose_url(self.api_url,
path=["lookml_models"],
params=params)
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-lookml",
title="Couldn't retrieve models and explores.",
status=response.status_code,
detail="Unable to retrieve LookML details. Please try again.",
response=response,
)
return response.json()
def build_project(
self,
selectors: Optional[List[str]] = None,
exclusions: Optional[List[str]] = None,
) -> None:
"""Creates an object representation of the project's LookML.
Args:
selectors: List of selector strings in 'model_name/explore_name' format.
The '*' wildcard selects all models or explores. For instance,
'model_name/*' would select all explores in the 'model_name' model.
"""
# Assign default values for selectors and exclusions
if selectors is None:
selectors = ["*/*"]
if exclusions is None:
exclusions = []
logger.info(
f"Building LookML project hierarchy for project {self.project.name}"
)
all_models = [
Model.from_json(model)
for model in self.client.get_lookml_models()
]
project_models = [
model for model in all_models
if model.project_name == self.project.name
]
if not project_models:
raise LookMlNotFound(
name="project-models-not-found",
title=
"No matching models found for the specified project and selectors.",
detail=(f"Go to {self.client.base_url}/projects and confirm "
"a) at least one model exists for the project and "
"b) it has an active configuration."),
)
for model in project_models:
model.explores = [
explore for explore in model.explores
if is_selected(model.name, explore.name, selectors, exclusions)
]
for explore in model.explores:
dimensions_json = self.client.get_lookml_dimensions(
model.name, explore.name)
for dimension_json in dimensions_json:
dimension = Dimension.from_json(dimension_json, model.name,
explore.name)
dimension.url = self.client.base_url + dimension.url
if not dimension.ignore:
explore.add_dimension(dimension)
self.project.models = [
model for model in project_models if len(model.explores) > 0
]
def get_lookml_dimensions(self, model: str, explore: str) -> List[str]:
"""Gets all dimensions for an explore from the LookmlModel endpoint.
Args:
model: Name of LookML model to query.
explore: Name of LookML explore to query.
Returns:
List[str]: Names of all the dimensions in the specified explore. Dimension
names are returned in the format 'explore_name.dimension_name'.
"""
logger.debug(f"Getting all dimensions from explore {explore}")
params = {"fields": ["fields"]}
url = utils.compose_url(
self.api_url,
path=["lookml_models", model, "explores", explore],
params=params,
)
response = self.get(url=url, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-get-dimension-lookml",
title="Couldn't retrieve dimensions.",
status=response.status_code,
detail=("Unable to retrieve dimension LookML details "
f"for explore '{model}/{explore}'. Please try again."),
response=response,
)
return response.json()["fields"]["dimensions"]
def _create_and_run(self, mode: QueryMode = "batch") -> None:
"""Runs a single validation using a specified mode"""
queries: List[Query] = []
try:
queries = self._create_queries(mode)
self._run_queries(queries)
except KeyboardInterrupt:
logger.info(
"\n\n" + "Please wait, asking Looker to cancel any running queries..."
)
query_tasks = self.get_running_query_tasks()
self._cancel_queries(query_tasks)
if query_tasks:
message = (
f"Attempted to cancel {len(query_tasks)} running "
f"{'query' if len(query_tasks) == 1 else 'queries'}."
)
else:
message = (
"No queries were running at the time so nothing was cancelled."
)
raise SpectaclesException(
name="validation-keyboard-interrupt",
title="SQL validation was manually interrupted.",
detail=message,
)
def update_workspace(self, project: str, workspace: str) -> None:
"""Updates the session workspace.
Args:
project: Name of the Looker project to use.
workspace: The workspace to switch to, either 'production' or 'dev'
"""
logger.debug(f"Updating session to use the {workspace} workspace")
url = utils.compose_url(self.api_url, path=["session"])
body = {"workspace_id": workspace}
response = self.patch(url=url, json=body, timeout=TIMEOUT_SEC)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
raise LookerApiError(
name="unable-to-update-workspace",
title="Couldn't update the session's workspace.",
status=response.status_code,
detail=(
f"Unable to update workspace to '{workspace}'. "
"If you have any unsaved work on the branch "
"checked out by the user whose API credentials "
"Spectacles is using, please save it and try again."
),
response=response,
)
