def generate_model(spoke_path: Path, name: str, namespace_defn: NamespaceDict) -> Path: """ Generate a model file for a namespace. We want these to have a nice label and a unique name. We only import explores and dashboards, as we want those to auto-import upon generation. Views are not imported by default, since they should be added one-by-one if they are included in an explore. """ logging.info(f"Generating model {name}...") model_defn = { "connection": "telemetry", "label": namespace_defn["pretty_name"], "includes": [ f"//looker-hub/{name}/explores/*", f"//looker-hub/{name}/dashboards/*", "views/*", "explores/*", "dashboards/*", ], } path = spoke_path / name / f"{name}.model.lkml" path.write_text(lkml.dump(model_defn)) return path
def assemble_view( cls, view_name: str, sql_table_name: str = None, derived_table: str = None, dimensions: List[dict] = None, dimension_groups: List[dict] = None, measures: List[dict] = None, sets: List[dict] = None, parameters: List[dict] = None, label: str = None, required_access_grants: list = None, extends: str = None, extension_is_required: bool = False, include_suggestions: bool = True, ): assembled_view_dict = {"view": {"name": view_name}} logger.info("Creating LookML View: {}".format(view_name)) # Validate inputs if not sql_table_name and not derived_table and not extends: raise DbteaException( name="missing-lookml-view-properties", title="Missing Necessary LookML View Properties", detail="Created LookML Views must specify either a `sql_table_name`, `derived_table` or `extends` in order " "to properly specify the view source", ) # Add optional view options as needed if label: assembled_view_dict["view"]["label"] = label if extends: assembled_view_dict["view"]["extends"] = extends if extension_is_required: assembled_view_dict["view"]["extension"] = "required" if sql_table_name: assembled_view_dict["view"]["sql_table_name"] = sql_table_name if derived_table: assembled_view_dict["view"]["derived_table"] = derived_table if required_access_grants: assembled_view_dict["view"][ "required_access_grants" ] = required_access_grants if not include_suggestions: assembled_view_dict["view"]["suggestions"] = "no" # Add body of View if parameters: assembled_view_dict["view"]["parameters"] = parameters if dimensions: assembled_view_dict["view"]["dimensions"] = dimensions if dimension_groups: assembled_view_dict["view"]["dimension_groups"] = dimension_groups if measures: assembled_view_dict["view"]["measures"] = measures if sets: assembled_view_dict["view"]["sets"] = sets return lkml.dump(assembled_view_dict)
def to_lookml( data: dict, output_to: str = "stdout", output_file: str = None, lookml_file_type: str = None, output_directory: str = None, ) -> Optional[str]: """""" if output_to == "stdout": return lkml.dump(data) else: output_file = ( utils.assemble_path( output_directory, output_file + "." + lookml_file_type + ".lkml" ) if lookml_file_type else utils.assemble_path(output_directory, output_file + ".lkml") ) with open(output_file, "w") as output_stream: output_stream.write(lkml.dump(data))
def test_model_with_all_fields(): path = Path( __file__).parent / "resources" / "model_with_all_fields.model.lkml" with path.open() as file: raw = file.read() parsed = lkml.load(raw) assert parsed is not None lookml = lkml.dump(parsed) assert lookml.replace("\n\n", "\n") == raw.replace("\n\n", "\n")
def lookml_view_from_dbt_model(model: models.DbtModel, adapter_type: models.SupportedDbtAdapters): lookml = { 'view': { 'name': model.name, 'sql_table_name': f'{model.database}.{model.db_schema}.{model.name}', 'dimension_groups': lookml_dimension_groups_from_model(model, adapter_type), 'dimensions': lookml_dimensions_from_model(model, adapter_type), } } contents = lkml.dump(lookml) filename = f'{model.name}.view' return models.LookViewFile(filename=filename, contents=contents)
def update_lkml(): LKML_PATH, DBT_PATH = cli() dbts, lkmls = load(LKML_PATH, DBT_PATH) for lk in lkmls: for dbt in dbts: if "sources" not in dbt.content: print(f"{lk.file_path} ----- {dbt.file_path}") comp = comparer(lk, dbt) if len(comp.lkml_to_add) > 0: with open(lk.file_path, "w") as f: f.write(lkml.dump(comp.new_lkml))
def lookml_model_from_dbt_project(dbt_models: List[models.DbtModel], dbt_project_name: str): lookml = { 'connection': dbt_project_name, 'include': '/views/*', 'explores': [ { 'name': model.name, 'description': model.description, } for model in dbt_models ] } contents = lkml.dump(lookml) filename = f'{dbt_project_name}.model' return models.LookModelFile(filename=filename, contents=contents)
def test_file(path: Path): with path.open("r") as file: text = file.read() try: parsed = lkml.load(text) except Exception: shutil.copy(path, github_path / "load_errors" / path.name) logging.exception(f"Error parsing {path}") try: dumped = lkml.dump(parsed) lkml.load(dumped) except Exception: with open(github_path / "dump_errors" / path.name, "w+") as file: file.write(dumped) logging.exception(f"Error serializing {path}")
####### # Step 3. opening the lkml file using the lkml library to serialize lookml into JSON with open(lookml, 'r') as file: parsed_view = lkml.load(file) dimensions = parsed_view['views'][0]['dimensions'] measures = parsed_view['views'][0]['measures'] dimension_groups = parsed_view['views'][0]['dimension_groups'] ####### # Step 4. Iterate over the serialized LookML and identifiying if there is a match between the henry unused dimensions and the lookml, add the hidden parameter to the JSON for y in range(len(parsed_view['views'])): for each in parsed_view['views'][y].keys(): values = parsed_view['views'][0][each] if each in ["dimensions", "measures", "dimension_groups"]: for number in range(len(values)): each2 = parsed_view['views'][0][each][number] if parsed_view['views'][0]['name'] + "." + each2[ 'name'] in uuf: parsed_view['views'][0][each][number]['hidden'] = "yes" print(each + ": " + parsed_view['views'][0]['name'] + "." + each2['name'] + " is unused so hiding...") ####### # Step 5. Write amended JSON back to LookML with open(lookml + '_out', 'w+') as outfile: lkml.dump(parsed_view, outfile) pp.pprint(parsed_view)
def create_lookml_model( model_name: str, output_to: str = "stdout", connection: str = None, label: str = None, includes: list = None, explores: List[dict] = None, access_grants: List[dict] = None, tests: List[dict] = None, datagroups: List[dict] = None, map_layers: List[dict] = None, named_value_formats: List[dict] = None, fiscal_month_offset: int = None, persist_for: str = None, persist_with: str = None, week_start_day: str = None, case_sensitive: bool = True, output_directory: str = None, ) -> Optional[str]: """""" assembled_model_dict = dict() logger.info("Creating LookML Model: {}".format(model_name)) # Validate inputs if output_to not in OUTPUT_TO_OPTIONS: raise DbteaException( name="invalid-lookml-model-properties", title="Invalid LookML Model Properties", detail="You must choose a valid output_to option from the following: {}".format( OUTPUT_TO_OPTIONS ), ) if output_to == "file" and not output_directory: raise DbteaException( name="missing-output-directory", title="No Model Output Directory Specified", detail="You must include an output_directory param if outputting model to a file", ) # Add optional model options if connection: assembled_model_dict["connection"] = connection if label: assembled_model_dict["label"] = label if includes: assembled_model_dict["includes"] = includes if persist_for: assembled_model_dict["persist_for"] = persist_for if persist_with: assembled_model_dict["persist_with"] = persist_with if fiscal_month_offset: assembled_model_dict["fiscal_month_offset"] = fiscal_month_offset if week_start_day: assembled_model_dict["week_start_day"] = week_start_day if not case_sensitive: assembled_model_dict["case_sensitive"] = "no" # Add body of Model if datagroups: assembled_model_dict["datagroups"] = datagroups if access_grants: assembled_model_dict["access_grants"] = access_grants if explores: assembled_model_dict["explores"] = explores if named_value_formats: assembled_model_dict["named_value_formats"] = named_value_formats if map_layers: assembled_model_dict["map_layers"] = map_layers if tests: assembled_model_dict["tests"] = tests if output_to == "stdout": return lkml.dump(assembled_model_dict) else: model_file_name = utils.assemble_path( output_directory, model_name + ".model.lkml" ) with open(model_file_name, "w") as output_stream: output_stream.write(lkml.dump(assembled_model_dict))
def lookml_string(self) -> str: """""" return lkml.dump(self.lookml_data)
path_to_lookml_file = os.path.join(PATH_TO_LOOKML_PROJECT, file_name) # load the lookml with open(path_to_lookml_file, "r") as file: lookml = lkml.load(file) # for each view in the lookml for view in lookml["views"]: view_name = view["name"] # for each dimension in the view for dimension in view["dimensions"]: dimension_name = dimension["name"] dimension_description = get_column_description( manifest, view_name, dimension_name) if dimension_description: # update the description based on the project's description dimension["description"] = dimension_description # dump the lmkl to a target directory os.makedirs(PATH_TO_TARGET_LOOKML_PROJECT, exist_ok=True) target_lookml_file = os.path.join(PATH_TO_TARGET_LOOKML_PROJECT, file_name) with open(target_lookml_file, "w+") as file: lkml.dump(lookml, file) # To-do: # - multiple lkml files # - parameterize the paths / add a cli # - Consider what is a reasonable assumption for matching views to models? the view name? # - error handling for unmatched view
def main(): start_time = time.time() # dest = input(f"What is your destination folder?") # local path to the PDF file from FiveTran path = open(f"/Users/jeffreymartinez/Downloads/Square ERD.pdf", 'rb') destination = f"/Users/jeffreymartinez/Desktop/block_flo/squaredesc" try: os.mkdir(destination, ) except OSError as error: print(error) print("Running...") urls = pdfurls(path) # Reads PDF, gives list of urls newurls = transurls(urls) # Takes urls, and gives valid square url syntax f = {} for url in newurls: tname, listt = tblmetadata( url ) # calls tblmetadata() which creates a dataframe of each documentation page tname = tname.lower() tname = tname.encode('ascii', 'ignore').decode( 'unicode_escape') # alters tname to avoid encoding errors f[tname] = listt.set_index( "fieldname", drop=False ) # maps each webpage as a dictionary of key: tablename, value: dataframe of fields/descriptions/types # print(tname) # prints each tablename found in online documentation # print(listt) viewfolder = f"/Users/jeffreymartinez/Desktop/block_flo/block-square/views" # Get folder of views to edit list_of_files = os.listdir(viewfolder) # creates list of view file names full_path = [viewfolder + "/{0}".format(x) for x in list_of_files] # gives full path of view files nomatch = [] count = 0 z = 0 for x in list_of_files: # iterate through each view file in the folder full_path = f"{viewfolder}/{x}" count += 1 s = open(full_path, "r+") k = lkml.load(s) webmatch = k['views'][0]['name'].replace( "_", "") # viewnames altered to match documentation titles (tnames) webmatch = webmatch.encode('ascii', 'ignore').decode('unicode_escape') print(f""" View name {k['views'][0]['name']} has the following fields: """) [ print(k['views'][0]['dimensions'][i]['name']) for i in range(0, len(k['views'][0]['dimensions'])) ] print(f""" """) try: link = f[webmatch] descriptions(link, k, z) z = z # print(link) except KeyError: print(f'No match found for view file named {webmatch}') # newmatch = input("Please input a new description tablename to try: ") # nlink = f[newmatch] # descriptions(nlink, k, z) nomatch.append(webmatch) pass h = open(f"{destination}/{x}", "w+") h.write(lkml.dump(k)) print(f""" Successfully added descriptions to {z} fields in {count - len(nomatch)} view files. No descriptions found for {len(nomatch)} of {count} total view files. List of files without descriptions: {nomatch}""") # print the program runtime print("--- %s seconds ---" % round(time.time() - start_time, 3))
def coremodel(sourcepath, destinationpath): blockname = sourcepath.split("/")[-1] print(f"Writing CORE & CONFIG Manifest files...") mfcore = open(f"{destinationpath}/CORE/manifest.lkml", "a+") mfcore.write(f"""project_name: "block-{blockname}" ################ Constants ################ constant: CONFIG_PROJECT_NAME {{ value: "block-{blockname}-config" export: override_required }} constant: CONNECTION_NAME {{ value: "choose connection" export: override_required }} ### If needed TODO Add more constants here ################ Dependencies ################ local_dependency: {{ project: "@{{CONFIG_PROJECT_NAME}}" #### If needed TODO Add CONFIG constants here that we want overridden by CORE constants }} """) mfconfig = open(f"{destinationpath}/CONFIG/manifest.lkml", "a+") mfconfig.write(f"""project_name: "block-{blockname}-config" ################ Constants ################ # If needed TODO Define constants with "export: override_required" declared """) # iterates through source directory for file in os.listdir(sourcepath): ogfilename = os.fsdecode(file) if len(ogfilename.split(".")) <= 1: continue else: # if the source file is a dashboard, we simply copy it over to the CORE if ogfilename.split(".")[1] == 'dashboard': shutil.copy2(f'{sourcepath}/{ogfilename}', f'{destinationpath}/CORE/{ogfilename}') # if the source file is the model, we have to take elements to build explores in CORE & CONFIG if ogfilename.split(".")[1] == 'model': f = open( f"{destinationpath}/CORE/block_{blockname}_{ogfilename}", 'a+') j = open(f"{destinationpath}/CONFIG/{blockname}_{ogfilename}", 'a+') print(f"Writing CORE model...") f.write(f"""connection: "@{{CONNECTION_NAME}}" include: "views/*.view.lkml" include: "*.explore.lkml" include: "*.dashboard.lookml" include: "//@{{CONFIG_PROJECT_NAME}}/*.view.lkml" include: "//@{{CONFIG_PROJECT_NAME}}/*.model.lkml" include: "//@{{CONFIG_PROJECT_NAME}}/*.dashboard" """) print(f"Writing CONFIG includes...") j.write(f"""include: "views/*.view" """) # opens up the source model file with open(f"{sourcepath}/{ogfilename}", 'r') as mfile: # uses lkml package to parse through the model file parsed = lkml.load(mfile) print(f"Parsing source model file...") print(f"Writing CORE & CONFIG explores...") for a in parsed['explores']: # For each source explore, we opens/creates a new file named by the explore, in the specifed destination given by argument g = open( f"{destinationpath}/CORE/{a['name']}.explore.lkml", 'w+') # Removes name key, as parser dump includes it for some reason b = a.pop("name") # Inside newly created explore file, we write the contents of the original explores g.write(f"""explore: {b}_core {{ extension: required {lkml.dump(a)} }}""") g.close() # Inside the new model file, we write the content layer explore! f.write(f"""explore: {b} {{ extends: [{b}_config] }} """) j.write(f"""explore: {b}_config {{ extends: [{b}_core] extension: required }} """) f.close() j.close() if ogfilename.split(".")[1] == 'view': vname = ogfilename.split(".")[0] h = open(f"""{destinationpath}/CORE/{vname}_core.view.lkml""", 'w+') d = open(f"""{destinationpath}/CONFIG/{ogfilename}""", 'w+') h.write( f"""include: "//@{{CONFIG_PROJECT_NAME}}/views/{ogfilename}" """) c = open(f"{sourcepath}/{ogfilename}", 'r') parsed = lkml.load(c) for g in range(len(parsed['views'])): defname = parsed['views'][g]['name'] h.write(f""" view: {defname} {{ extends: [{defname}_config] }} """) d.write(f"""view: {defname}_config {{ extends: [{defname}_core] extension: required # Add view customizations here }} """) parsed['views'][g][ 'name'] = f"{parsed['views'][g]['name']}_core" contents = lkml.dump(parsed) h.write(f"""################################################### """) h.write(contents) print(f"Writing CORE & CONFIG view files...") return