def run_cli_command(
command: Union[List[str], str],
working_directory: str,
use_shell: bool = False,
output_as_text: bool = True,
capture_output: bool = True,
**kwargs,
):
"""Execute command line subprocess"""
result = subprocess.run(
command,
shell=use_shell,
cwd=working_directory,
text=output_as_text,
capture_output=capture_output,
**kwargs,
)
if result.stderr:
raise subprocess.CalledProcessError(returncode=result.returncode,
cmd=result.args,
stderr=result.stderr)
if result.stdout:
if "Encountered an error" in result.stdout: # Handle for dbt stdout errors
logger.error("dbt Error: {}".format(result.stdout))
raise subprocess.CalledProcessError(returncode=result.returncode,
cmd=result.args,
stderr=result.stdout)
logger.debug("Command Result:\n{}".format(result.stdout))
return result.stdout
def dbt_model_schemas_to_lookml_views(dbt_models_data: List[dict]) -> dict:
""""""
lookml_views_dbt_data = dbt_models_data
for model_data in lookml_views_dbt_data:
if "columns" in model_data:
model_data["dimensions"] = list()
model_data["dimension_groups"] = list()
invalid_column_properties = list()
for index, column_data in enumerate(model_data["columns"]):
for column_property_key in column_data.keys():
if column_property_key not in VALID_LOOKML_DIMENSION_PROPERTIES:
invalid_column_properties.append(column_property_key)
for invalid_column_property in invalid_column_properties:
logger.debug(
"Removing property invalid for LookML for dimension {}: {}".format(
model_data.get("name"), invalid_column_property
)
)
if model_data["columns"][index].get(invalid_column_property):
del model_data["columns"][index][invalid_column_property]
if column_data.get("type") in LOOKML_DIMENSION_GROUP_TYPES:
column_data.update({"sql": "${TABLE}." + column_data.get("name")})
model_data["dimension_groups"].append(column_data)
else:
column_data.update({"sql": "${TABLE}." + column_data.get("name")})
model_data["dimensions"].append(column_data)
# model_data["dimensions"][index]["sql"] = "${TABLE}." + column_data.get("name")
del model_data["columns"]
if "alias" in model_data:
model_data["sql_table_name"] = model_data["alias"]
else:
model_data["sql_table_name"] = model_data.get(
"name"
) # TODO: Needs to be fully qualified name]
# Process model-level meta fields
if "meta" in model_data:
for meta_key, meta_value in model_data["meta"].items():
if meta_key[:7] == "looker_" or meta_key[:7] == "lookml_":
model_data[meta_key[7:]] = model_data["meta"][meta_key]
invalid_properties = list()
for model_property in model_data.keys():
if model_property not in VALID_LOOKML_VIEW_PROPERTIES:
invalid_properties.append(model_property)
for invalid_property in invalid_properties:
logger.debug(
"Removing property invalid for LookML for view {}: {}".format(
model_data.get("name"), invalid_property
)
)
del model_data[invalid_property]
return {"views": lookml_views_dbt_data}
def wrapper(*args, **kwargs):
try:
return function(*args, **kwargs)
except DbteaException as error:
logger.error(
f"\n{error}\n\n" +
"For support, please create an issue at https://github.com/spectacles-ci/spectacles/issues"
+ "\n")
sys.exit(error.exit_code)
except KeyboardInterrupt as error:
logger.debug(error, exc_info=True)
logger.info("Spectacles was manually interrupted.")
sys.exit(1)
except Exception as error:
logger.debug(error, exc_info=True)
logger.error(
f'\nEncountered unexpected {error.__class__.__name__}: "{error}"\n'
f"Full error traceback logged to file.\n\n" +
"For support, please create an issue at https://github.com/spectacles-ci/spectacles/issues"
+ "\n")
sys.exit(1)
def convert_to_lookml_data_type(field_name: str,
field_type: str,
include_postal_code: bool = False) -> str:
""""""
if include_postal_code and field_name in LOOKML_ZIPCODE_FIELD_NAMES:
lookml_data_type = LOOKML_TYPE_ZIP
elif field_type in LOOKML_YESNO_DATA_TYPES:
lookml_data_type = LOOKML_TYPE_BOOL
elif field_type in LOOKML_TIME_DATA_TYPES:
lookml_data_type = LOOKML_TYPE_DATETIME
elif field_type in LOOKML_DATE_DATA_TYPES:
lookml_data_type = LOOKML_TYPE_DATE
elif field_type in LOOKML_NUMBER_DATA_TYPES:
lookml_data_type = LOOKML_TYPE_NUMBER
else:
lookml_data_type = LOOKML_TYPE_STRING
logger.debug(
f"Field: {field_name} with data type: {field_type} was mapped to LookML type: {lookml_data_type}"
)
return lookml_data_type