def generate_model(spoke_path: Path, name: str, namespace_defn: NamespaceDict) -> Path:
"""
Generate a model file for a namespace.
We want these to have a nice label and a unique name.
We only import explores and dashboards, as we want those
to auto-import upon generation.
Views are not imported by default, since they should
be added one-by-one if they are included in an explore.
"""
logging.info(f"Generating model {name}...")
model_defn = {
"connection": "telemetry",
"label": namespace_defn["pretty_name"],
"includes": [
f"//looker-hub/{name}/explores/*",
f"//looker-hub/{name}/dashboards/*",
"views/*",
"explores/*",
"dashboards/*",
],
}
path = spoke_path / name / f"{name}.model.lkml"
path.write_text(lkml.dump(model_defn))
return path
def assemble_view(
cls,
view_name: str,
sql_table_name: str = None,
derived_table: str = None,
dimensions: List[dict] = None,
dimension_groups: List[dict] = None,
measures: List[dict] = None,
sets: List[dict] = None,
parameters: List[dict] = None,
label: str = None,
required_access_grants: list = None,
extends: str = None,
extension_is_required: bool = False,
include_suggestions: bool = True,
):
assembled_view_dict = {"view": {"name": view_name}}
logger.info("Creating LookML View: {}".format(view_name))
# Validate inputs
if not sql_table_name and not derived_table and not extends:
raise DbteaException(
name="missing-lookml-view-properties",
title="Missing Necessary LookML View Properties",
detail="Created LookML Views must specify either a `sql_table_name`, `derived_table` or `extends` in order "
"to properly specify the view source",
)
# Add optional view options as needed
if label:
assembled_view_dict["view"]["label"] = label
if extends:
assembled_view_dict["view"]["extends"] = extends
if extension_is_required:
assembled_view_dict["view"]["extension"] = "required"
if sql_table_name:
assembled_view_dict["view"]["sql_table_name"] = sql_table_name
if derived_table:
assembled_view_dict["view"]["derived_table"] = derived_table
if required_access_grants:
assembled_view_dict["view"][
"required_access_grants"
] = required_access_grants
if not include_suggestions:
assembled_view_dict["view"]["suggestions"] = "no"
# Add body of View
if parameters:
assembled_view_dict["view"]["parameters"] = parameters
if dimensions:
assembled_view_dict["view"]["dimensions"] = dimensions
if dimension_groups:
assembled_view_dict["view"]["dimension_groups"] = dimension_groups
if measures:
assembled_view_dict["view"]["measures"] = measures
if sets:
assembled_view_dict["view"]["sets"] = sets
return lkml.dump(assembled_view_dict)
def to_lookml(
data: dict,
output_to: str = "stdout",
output_file: str = None,
lookml_file_type: str = None,
output_directory: str = None,
) -> Optional[str]:
""""""
if output_to == "stdout":
return lkml.dump(data)
else:
output_file = (
utils.assemble_path(
output_directory, output_file + "." + lookml_file_type + ".lkml"
)
if lookml_file_type
else utils.assemble_path(output_directory, output_file + ".lkml")
)
with open(output_file, "w") as output_stream:
output_stream.write(lkml.dump(data))
def test_model_with_all_fields():
path = Path(
__file__).parent / "resources" / "model_with_all_fields.model.lkml"
with path.open() as file:
raw = file.read()
parsed = lkml.load(raw)
assert parsed is not None
lookml = lkml.dump(parsed)
assert lookml.replace("\n\n", "\n") == raw.replace("\n\n", "\n")
def lookml_view_from_dbt_model(model: models.DbtModel, adapter_type: models.SupportedDbtAdapters):
lookml = {
'view': {
'name': model.name,
'sql_table_name': f'{model.database}.{model.db_schema}.{model.name}',
'dimension_groups': lookml_dimension_groups_from_model(model, adapter_type),
'dimensions': lookml_dimensions_from_model(model, adapter_type),
}
}
contents = lkml.dump(lookml)
filename = f'{model.name}.view'
return models.LookViewFile(filename=filename, contents=contents)
def update_lkml():
LKML_PATH, DBT_PATH = cli()
dbts, lkmls = load(LKML_PATH, DBT_PATH)
for lk in lkmls:
for dbt in dbts:
if "sources" not in dbt.content:
print(f"{lk.file_path} ----- {dbt.file_path}")
comp = comparer(lk, dbt)
if len(comp.lkml_to_add) > 0:
with open(lk.file_path, "w") as f:
f.write(lkml.dump(comp.new_lkml))
def lookml_model_from_dbt_project(dbt_models: List[models.DbtModel], dbt_project_name: str):
lookml = {
'connection': dbt_project_name,
'include': '/views/*',
'explores': [
{
'name': model.name,
'description': model.description,
}
for model in dbt_models
]
}
contents = lkml.dump(lookml)
filename = f'{dbt_project_name}.model'
return models.LookModelFile(filename=filename, contents=contents)
def test_file(path: Path):
with path.open("r") as file:
text = file.read()
try:
parsed = lkml.load(text)
except Exception:
shutil.copy(path, github_path / "load_errors" / path.name)
logging.exception(f"Error parsing {path}")
try:
dumped = lkml.dump(parsed)
lkml.load(dumped)
except Exception:
with open(github_path / "dump_errors" / path.name, "w+") as file:
file.write(dumped)
logging.exception(f"Error serializing {path}")
#######
# Step 3. opening the lkml file using the lkml library to serialize lookml into JSON
with open(lookml, 'r') as file:
parsed_view = lkml.load(file)
dimensions = parsed_view['views'][0]['dimensions']
measures = parsed_view['views'][0]['measures']
dimension_groups = parsed_view['views'][0]['dimension_groups']
#######
# Step 4. Iterate over the serialized LookML and identifiying if there is a match between the henry unused dimensions and the lookml, add the hidden parameter to the JSON
for y in range(len(parsed_view['views'])):
for each in parsed_view['views'][y].keys():
values = parsed_view['views'][0][each]
if each in ["dimensions", "measures", "dimension_groups"]:
for number in range(len(values)):
each2 = parsed_view['views'][0][each][number]
if parsed_view['views'][0]['name'] + "." + each2[
'name'] in uuf:
parsed_view['views'][0][each][number]['hidden'] = "yes"
print(each + ": " + parsed_view['views'][0]['name'] +
"." + each2['name'] + " is unused so hiding...")
#######
# Step 5. Write amended JSON back to LookML
with open(lookml + '_out', 'w+') as outfile:
lkml.dump(parsed_view, outfile)
pp.pprint(parsed_view)
def create_lookml_model(
model_name: str,
output_to: str = "stdout",
connection: str = None,
label: str = None,
includes: list = None,
explores: List[dict] = None,
access_grants: List[dict] = None,
tests: List[dict] = None,
datagroups: List[dict] = None,
map_layers: List[dict] = None,
named_value_formats: List[dict] = None,
fiscal_month_offset: int = None,
persist_for: str = None,
persist_with: str = None,
week_start_day: str = None,
case_sensitive: bool = True,
output_directory: str = None,
) -> Optional[str]:
""""""
assembled_model_dict = dict()
logger.info("Creating LookML Model: {}".format(model_name))
# Validate inputs
if output_to not in OUTPUT_TO_OPTIONS:
raise DbteaException(
name="invalid-lookml-model-properties",
title="Invalid LookML Model Properties",
detail="You must choose a valid output_to option from the following: {}".format(
OUTPUT_TO_OPTIONS
),
)
if output_to == "file" and not output_directory:
raise DbteaException(
name="missing-output-directory",
title="No Model Output Directory Specified",
detail="You must include an output_directory param if outputting model to a file",
)
# Add optional model options
if connection:
assembled_model_dict["connection"] = connection
if label:
assembled_model_dict["label"] = label
if includes:
assembled_model_dict["includes"] = includes
if persist_for:
assembled_model_dict["persist_for"] = persist_for
if persist_with:
assembled_model_dict["persist_with"] = persist_with
if fiscal_month_offset:
assembled_model_dict["fiscal_month_offset"] = fiscal_month_offset
if week_start_day:
assembled_model_dict["week_start_day"] = week_start_day
if not case_sensitive:
assembled_model_dict["case_sensitive"] = "no"
# Add body of Model
if datagroups:
assembled_model_dict["datagroups"] = datagroups
if access_grants:
assembled_model_dict["access_grants"] = access_grants
if explores:
assembled_model_dict["explores"] = explores
if named_value_formats:
assembled_model_dict["named_value_formats"] = named_value_formats
if map_layers:
assembled_model_dict["map_layers"] = map_layers
if tests:
assembled_model_dict["tests"] = tests
if output_to == "stdout":
return lkml.dump(assembled_model_dict)
else:
model_file_name = utils.assemble_path(
output_directory, model_name + ".model.lkml"
)
with open(model_file_name, "w") as output_stream:
output_stream.write(lkml.dump(assembled_model_dict))
def lookml_string(self) -> str:
""""""
return lkml.dump(self.lookml_data)
path_to_lookml_file = os.path.join(PATH_TO_LOOKML_PROJECT, file_name)
# load the lookml
with open(path_to_lookml_file, "r") as file:
lookml = lkml.load(file)
# for each view in the lookml
for view in lookml["views"]:
view_name = view["name"]
# for each dimension in the view
for dimension in view["dimensions"]:
dimension_name = dimension["name"]
dimension_description = get_column_description(
manifest, view_name, dimension_name)
if dimension_description:
# update the description based on the project's description
dimension["description"] = dimension_description
# dump the lmkl to a target directory
os.makedirs(PATH_TO_TARGET_LOOKML_PROJECT, exist_ok=True)
target_lookml_file = os.path.join(PATH_TO_TARGET_LOOKML_PROJECT, file_name)
with open(target_lookml_file, "w+") as file:
lkml.dump(lookml, file)
# To-do:
# - multiple lkml files
# - parameterize the paths / add a cli
# - Consider what is a reasonable assumption for matching views to models? the view name?
# - error handling for unmatched view
def main():
start_time = time.time()
# dest = input(f"What is your destination folder?")
# local path to the PDF file from FiveTran
path = open(f"/Users/jeffreymartinez/Downloads/Square ERD.pdf", 'rb')
destination = f"/Users/jeffreymartinez/Desktop/block_flo/squaredesc"
try:
os.mkdir(destination, )
except OSError as error:
print(error)
print("Running...")
urls = pdfurls(path) # Reads PDF, gives list of urls
newurls = transurls(urls) # Takes urls, and gives valid square url syntax
f = {}
for url in newurls:
tname, listt = tblmetadata(
url
) # calls tblmetadata() which creates a dataframe of each documentation page
tname = tname.lower()
tname = tname.encode('ascii', 'ignore').decode(
'unicode_escape') # alters tname to avoid encoding errors
f[tname] = listt.set_index(
"fieldname", drop=False
) # maps each webpage as a dictionary of key: tablename, value: dataframe of fields/descriptions/types
# print(tname) # prints each tablename found in online documentation
# print(listt)
viewfolder = f"/Users/jeffreymartinez/Desktop/block_flo/block-square/views" # Get folder of views to edit
list_of_files = os.listdir(viewfolder) # creates list of view file names
full_path = [viewfolder + "/{0}".format(x)
for x in list_of_files] # gives full path of view files
nomatch = []
count = 0
z = 0
for x in list_of_files: # iterate through each view file in the folder
full_path = f"{viewfolder}/{x}"
count += 1
s = open(full_path, "r+")
k = lkml.load(s)
webmatch = k['views'][0]['name'].replace(
"_",
"") # viewnames altered to match documentation titles (tnames)
webmatch = webmatch.encode('ascii', 'ignore').decode('unicode_escape')
print(f"""
View name {k['views'][0]['name']} has the following fields:
""")
[
print(k['views'][0]['dimensions'][i]['name'])
for i in range(0, len(k['views'][0]['dimensions']))
]
print(f"""
""")
try:
link = f[webmatch]
descriptions(link, k, z)
z = z
# print(link)
except KeyError:
print(f'No match found for view file named {webmatch}')
# newmatch = input("Please input a new description tablename to try: ")
# nlink = f[newmatch]
# descriptions(nlink, k, z)
nomatch.append(webmatch)
pass
h = open(f"{destination}/{x}", "w+")
h.write(lkml.dump(k))
print(f"""
Successfully added descriptions to {z} fields in {count - len(nomatch)} view files. No descriptions found for {len(nomatch)} of {count} total view files. List of files without descriptions:
{nomatch}""")
# print the program runtime
print("--- %s seconds ---" % round(time.time() - start_time, 3))
def coremodel(sourcepath, destinationpath):
blockname = sourcepath.split("/")[-1]
print(f"Writing CORE & CONFIG Manifest files...")
mfcore = open(f"{destinationpath}/CORE/manifest.lkml", "a+")
mfcore.write(f"""project_name: "block-{blockname}"
################ Constants ################
constant: CONFIG_PROJECT_NAME {{
value: "block-{blockname}-config"
export: override_required
}}
constant: CONNECTION_NAME {{
value: "choose connection"
export: override_required
}}
### If needed TODO Add more constants here
################ Dependencies ################
local_dependency: {{
project: "@{{CONFIG_PROJECT_NAME}}"
#### If needed TODO Add CONFIG constants here that we want overridden by CORE constants
}}
""")
mfconfig = open(f"{destinationpath}/CONFIG/manifest.lkml", "a+")
mfconfig.write(f"""project_name: "block-{blockname}-config"
################ Constants ################
# If needed TODO Define constants with "export: override_required" declared
""")
# iterates through source directory
for file in os.listdir(sourcepath):
ogfilename = os.fsdecode(file)
if len(ogfilename.split(".")) <= 1:
continue
else:
# if the source file is a dashboard, we simply copy it over to the CORE
if ogfilename.split(".")[1] == 'dashboard':
shutil.copy2(f'{sourcepath}/{ogfilename}',
f'{destinationpath}/CORE/{ogfilename}')
# if the source file is the model, we have to take elements to build explores in CORE & CONFIG
if ogfilename.split(".")[1] == 'model':
f = open(
f"{destinationpath}/CORE/block_{blockname}_{ogfilename}",
'a+')
j = open(f"{destinationpath}/CONFIG/{blockname}_{ogfilename}",
'a+')
print(f"Writing CORE model...")
f.write(f"""connection: "@{{CONNECTION_NAME}}"
include: "views/*.view.lkml"
include: "*.explore.lkml"
include: "*.dashboard.lookml"
include: "//@{{CONFIG_PROJECT_NAME}}/*.view.lkml"
include: "//@{{CONFIG_PROJECT_NAME}}/*.model.lkml"
include: "//@{{CONFIG_PROJECT_NAME}}/*.dashboard"
""")
print(f"Writing CONFIG includes...")
j.write(f"""include: "views/*.view"
""")
# opens up the source model file
with open(f"{sourcepath}/{ogfilename}", 'r') as mfile:
# uses lkml package to parse through the model file
parsed = lkml.load(mfile)
print(f"Parsing source model file...")
print(f"Writing CORE & CONFIG explores...")
for a in parsed['explores']:
# For each source explore, we opens/creates a new file named by the explore, in the specifed destination given by argument
g = open(
f"{destinationpath}/CORE/{a['name']}.explore.lkml",
'w+')
# Removes name key, as parser dump includes it for some reason
b = a.pop("name")
# Inside newly created explore file, we write the contents of the original explores
g.write(f"""explore: {b}_core {{
extension: required
{lkml.dump(a)}
}}""")
g.close()
# Inside the new model file, we write the content layer explore!
f.write(f"""explore: {b} {{
extends: [{b}_config]
}}
""")
j.write(f"""explore: {b}_config {{
extends: [{b}_core]
extension: required
}}
""")
f.close()
j.close()
if ogfilename.split(".")[1] == 'view':
vname = ogfilename.split(".")[0]
h = open(f"""{destinationpath}/CORE/{vname}_core.view.lkml""",
'w+')
d = open(f"""{destinationpath}/CONFIG/{ogfilename}""", 'w+')
h.write(
f"""include: "//@{{CONFIG_PROJECT_NAME}}/views/{ogfilename}"
""")
c = open(f"{sourcepath}/{ogfilename}", 'r')
parsed = lkml.load(c)
for g in range(len(parsed['views'])):
defname = parsed['views'][g]['name']
h.write(f"""
view: {defname} {{
extends: [{defname}_config]
}}
""")
d.write(f"""view: {defname}_config {{
extends: [{defname}_core]
extension: required
# Add view customizations here
}}
""")
parsed['views'][g][
'name'] = f"{parsed['views'][g]['name']}_core"
contents = lkml.dump(parsed)
h.write(f"""###################################################
""")
h.write(contents)
print(f"Writing CORE & CONFIG view files...")
return
