def parse_lookml_file(lookml_file_name: str) -> dict:
"""Parse a LookML file into a dictionary with keys for each of its primary properties and a list of values."""
logger.info("Parsing data from LookML file {}".format(lookml_file_name))
with open(lookml_file_name, "r") as lookml_file_stream:
lookml_data = lkml.load(lookml_file_stream)
return lookml_data
def _load_viewfile(
self, path: str,
reporter: LookMLSourceReport) -> Optional[LookerViewFile]:
if self.is_view_seen(path):
return self.viewfile_cache[path]
try:
with open(path, "r") as file:
raw_file_content = file.read()
except Exception as e:
self.reporter.report_failure(path,
f"failed to load view file: {e}")
return None
try:
with open(path, "r") as file:
logger.info(f"Loading file {path}")
parsed = lkml.load(file)
looker_viewfile = LookerViewFile.from_looker_dict(
absolute_file_path=path,
looker_view_file_dict=parsed,
base_folder=self._base_folder,
raw_file_content=raw_file_content,
reporter=reporter,
)
logger.debug(f"adding viewfile for path {path} to the cache")
self.viewfile_cache[path] = looker_viewfile
return looker_viewfile
except Exception as e:
self.reporter.report_failure(path,
f"failed to load view file: {e}")
return None
def test_parser():
lookml = {
"explores": [
{
"name": "test_explore",
"joins": [
{
"name": "join_a"
},
{
"name": "join_b"
},
],
"queries": {
"dimensions": ["submission_date", "app_build"],
"measures": ["client_count"],
"pivots": ["app_build"],
"sorts": [{
"submission_date": "asc"
}],
"name": "build_breakdown",
},
},
],
}
print_and_test(lookml, lkml.load(dump(copy.deepcopy(lookml))))
def _load_model(self, path: str) -> LookerModel:
with open(path, "r") as file:
parsed = lkml.load(file)
looker_model = LookerModel.from_looker_dict(
parsed, self.source_config.base_folder
)
return looker_model
def __init__(self, infilepath):
'''parse the LookML infilepath, convert to JSON, and then read into JSON object
Args:
infilepath (str): path to input LookML file
Returns:
JSON object of LookML
'''
if not os.path.exists(infilepath):
raise IOError("Filename does not exist: %s" % infilepath)
self.infilepath = infilepath
if infilepath.endswith(".model.lkml"):
self.filetype = 'model'
elif infilepath.endswith(".view.lkml"):
self.filetype = 'view'
elif infilepath.endswith(".explore.lkml"):
self.filetype = 'explore'
else:
raise Exception("Unsupported filename " + infilepath)
self.base_filename = os.path.basename(infilepath)
self.base_name = self.base_filename.replace(".model.lkml", "").replace(
".explore.lkml", "").replace(".view.lkml", "")
with open(infilepath, 'r') as file:
self.json_data = lkml.load(file)
def github_lkml(repo_name: str):
"""connect to repo with lookml, iterate through lookml and returns list
of parsed lookml
Args:
repo_name (str): name of repo (not path)
Returns:
[list]: list of lookml elements in repo
"""
g = Github(github_token)
repo = g.get_repo(repo_name)
contents = repo.get_contents("", ref='master')
print(contents)
lookml = []
while contents:
file_content = contents.pop(0)
if file_content.type == "dir":
contents.extend(repo.get_contents(file_content.path))
else:
x = base64.b64decode(file_content.content).decode('UTF-8')
try:
lookml.append(lkml.load(x))
except SyntaxError:
print(f'''you have a lookml syntax error in {file_content}
and your file cannot be parsed
''')
return lookml
def test_generate_model(looker_sdk, namespaces, tmp_path):
sdk = looker_sdk.init31()
sdk.search_model_sets.side_effect = [
[Mock(models=["model"], id=1)],
[Mock(models=["model2"], id=2)],
]
sdk.lookml_model.side_effect = _looker_sdk.error.SDKError
looker_sdk.error = Mock(SDKError=_looker_sdk.error.SDKError)
write_model = Mock()
looker_sdk.models.WriteModelSet.return_value = write_model
generate_directories(namespaces, tmp_path, True)
expected = {
"connection":
"telemetry",
"label":
"Glean App",
"includes": [
"//looker-hub/glean-app/explores/*",
"//looker-hub/glean-app/dashboards/*",
"views/*",
"explores/*",
"dashboards/*",
],
}
actual = lkml.load(
(tmp_path / "glean-app" / "glean-app.model.lkml").read_text())
assert expected == actual
sdk.update_model_set.assert_any_call(1, write_model)
sdk.update_model_set.assert_any_call(2, write_model)
def __init__(self, infilepath):
"""parse the LookML infilepath, convert to JSON, and then read into JSON object
Args:
infilepath (str): path to input LookML file
Returns:
JSON object of LookML
"""
if not os.path.exists(infilepath):
raise IOError("Filename does not exist: %s" % infilepath)
self.infilepath = infilepath
self.base_filename = os.path.basename(infilepath)
if self.base_filename.endswith(".model.lkml"):
self.filetype = "model"
elif self.base_filename.endswith(".explore.lkml"):
self.filetype = "explore"
elif self.base_filename.startswith(
"explore_") and self.base_filename.endswith(".lkml"):
self.filetype = "explore"
elif self.base_filename.endswith(".view.lkml"):
self.filetype = "view"
elif self.base_filename.endswith(".lkml"):
# consider everything else as a view
self.filetype = "view"
else:
raise Exception("Unsupported filename " + infilepath)
self.base_name = re.sub(r"\.\w+\.lkml$", "", self.base_filename)
with open(infilepath, "r") as file:
self.json_data = lkml.load(file)
def getViewInformationFromFile(self,
fileName,
isExtended=False,
logging=None):
if logging is None:
logging = Logger().getLogger()
views = []
with open(fileName, 'r') as file:
parsed = lkml.load(file)
#print(parsed)
logging.info(parsed)
includes = []
if 'includes' in parsed:
includes = parsed['includes']
for view in parsed['views']:
viewObj = View()
viewObj.setView(view)
viewObj.includes = includes
if isExtended:
viewObj.viewType = 'EXTENDED'
views.append(viewObj)
return views
def parse(self):
"""
open file and load lkml as json
"""
with open(self.file_path, "r") as f:
self.content = lkml.load(f)
return self.content
def _load_model(self, path: str) -> LookerModel:
with open(path, "r") as file:
logger.debug(f"Loading model from file {path}")
parsed = lkml.load(file)
looker_model = LookerModel.from_looker_dict(
parsed, str(self.source_config.base_folder), path,
self.reporter)
return looker_model
def test_file(path: Path):
with path.open("r") as file:
text = file.read()
try:
parsed = lkml.load(text)
except Exception:
shutil.copy(path, github_path / "load_errors" / path.name)
logging.exception(f"Error parsing {path}")
try:
dumped = lkml.dump(parsed)
lkml.load(dumped)
except Exception:
with open(github_path / "dump_errors" / path.name, "w+") as file:
file.write(dumped)
logging.exception(f"Error serializing {path}")
def test_round_trip_should_work(lookml):
# Load the LookML from file, parsing into a tree
tree = lkml.parse(lookml)
# Verify it hasn't changed once converted back to string
assert str(tree) == lookml
# Convert that parsed tree into a lossy dictionary
visitor = DictVisitor()
tree_as_dict = visitor.visit(tree)
# Parse the dictionary into a new tree
parser = DictParser()
new_tree = parser.parse(tree_as_dict)
# Verify that the string form of the tree parsed from a dictionary can be re-parsed
lkml.load(str(new_tree))
def test_model_with_all_fields():
path = Path(
__file__).parent / "resources" / "model_with_all_fields.model.lkml"
with path.open() as file:
raw = file.read()
parsed = lkml.load(raw)
assert parsed is not None
def test_model_with_all_fields():
path = Path(
__file__).parent / "resources" / "model_with_all_fields.model.lkml"
with path.open() as file:
raw = file.read()
parsed = lkml.load(raw)
assert parsed is not None
lookml = lkml.dump(parsed)
assert lookml.replace("\n\n", "\n") == raw.replace("\n\n", "\n")
def _load_viewfile(self, path: str) -> typing.Optional["LookerViewFile"]:
if path in self.viewfile_cache:
return self.viewfile_cache[path]
try:
with open(path, "r") as file:
parsed = lkml.load(file)
looker_viewfile = LookerViewFile.from_looker_dict(path, parsed)
self.viewfile_cache[path] = looker_viewfile
return looker_viewfile
except Exception as e:
print(e)
print(f"Error processing view file {path}. Skipping it")
def _load_viewfile(self, path: str) -> Optional[LookerViewFile]:
if self.is_view_seen(path):
return self.viewfile_cache[path]
try:
with open(path, "r") as file:
parsed = lkml.load(file)
looker_viewfile = LookerViewFile.from_looker_dict(
path, parsed, self._base_folder)
self.viewfile_cache[path] = looker_viewfile
return looker_viewfile
except Exception:
logger.warning(f"Error processing view file {path}. Skipping it")
return None
def parse(file):
print('[parsing] {}'.format(f))
try:
with open(f, 'r') as file:
parsed = lkml.load(file)
if 'explore' in parsed.keys():
find_joins(parsed['explore'])
elif 'explores' in parsed.keys():
for explore in parsed['explores']:
find_joins(explore)
except Exception as e:
print('[parser error] {}'.format(f))
print(traceback.format_exc())
def from_lookml_string(
cls,
name: str,
lookml_type: str,
directory_path: str = "./",
lookml_string: str = None,
):
""""""
return cls(
name,
lookml_type,
directory_path=directory_path,
lookml_data=lkml.load(lookml_string),
)
def read_data_from_file(self, local_lookml_project_path: str) -> dict:
"""Parse a LookML file into a dictionary with keys for each of its primary properties and a list of values."""
logger.info(
"Parsing data from local LookML file {}".format(
self.lookml_file_name_and_path
)
)
with open(
utils.assemble_path(
local_lookml_project_path, self.lookml_file_name_and_path
),
"r",
) as lookml_file:
return lkml.load(lookml_file)
def __attrs_post_init__(self):
repo_path = Path(self.lookml_repo_path)
self.models = []
for model_file in self._model_file_paths():
with open(repo_path.joinpath(model_file)) as f:
self.models.append(
Model(lkml.load(f), os.path.basename(model_file)))
self.views = []
for view_file in self._view_file_paths():
with open(repo_path.joinpath(view_file)) as f:
for view_data in lkml.load(f)['views']:
self.views.append(
View(view_data, os.path.basename(view_file)))
# match explore views with their source views
for m in self.models:
for e in m.explores:
for ev in e.views:
source_view = next(v for v in self.views
if v.name == ev.source_view_name())
ev.source_view = source_view
def git_auth(user, token, repo, file_name, owner):
query_url = f"https://api.github.com/repos/{owner}/{repo}"
params = {
"state": "open",
}
headers = {'Authorization': f'token {token}'}
r = requests.get(query_url, headers=headers, params=params)
#grab github repo content(s)
g = Github(token)
repo = g.get_repo(owner + "/" + repo)
#specify lookml file
content = repo.get_contents(path="/Views/" + file_name)
raw_data = content.decoded_content.decode("utf-8")
parsed_lookml = lkml.load(raw_data)
return parsed_lookml
def file_parse_lkml(file_paths: list):
"""Iterate through local file paths
and return a list of lookml files with an element
for each lookml file]
Args:
file_paths (list): [a list of local path strings]
Returns:
[list]: [list of parsed lookml]
"""
response_list = []
for path in file_paths:
with open(path, 'r') as file:
response_list.append(lkml.load(file))
return response_list
def from_local_file(cls, looker_project_root: str, file_relative_path: str):
""""""
full_project_path = utils.assemble_path(looker_project_root, file_relative_path)
path_suffixes = Path(full_project_path).suffixes
lookml_file_type = path_suffixes[0] if len(path_suffixes) > 1 else "generic"
directory, file_name = (
Path(full_project_path).parent,
Path(full_project_path).stem,
)
with open(full_project_path, "r") as local_file_stream:
return cls(
file_name,
lookml_file_type,
directory_path=directory,
lookml_data=lkml.load(local_file_stream),
)
def getViewInfomationFromFile(self, fileName):
views = []
with open(fileName, 'r') as file:
parsed = lkml.load(file)
print(parsed)
#logging.info(parsed)
for view in parsed['views']:
viewObj = View()
viewObj.setView(view)
views.append(viewObj)
return views
def _load_viewfile(
self, path: str,
reporter: LookMLSourceReport) -> Optional[LookerViewFile]:
if self.is_view_seen(path):
return self.viewfile_cache[path]
try:
with open(path, "r") as file:
parsed = lkml.load(file)
looker_viewfile = LookerViewFile.from_looker_dict(
path, parsed, self._base_folder, reporter)
logger.debug(f"adding viewfile for path {path} to the cache")
self.viewfile_cache[path] = looker_viewfile
return looker_viewfile
except Exception as e:
self.reporter.report_failure(path,
f"failed to load view file: {e}")
return None
def _load_viewfile(
self, path: str,
reporter: LookMLSourceReport) -> Optional[LookerViewFile]:
# always fully resolve paths to simplify de-dup
path = str(pathlib.Path(path).resolve())
if not path.endswith(".view.lkml"):
# not a view file
logger.debug(
f"Skipping file {path} because it doesn't appear to be a view file"
)
return None
if self.is_view_seen(str(path)):
return self.viewfile_cache[path]
try:
with open(path, "r") as file:
raw_file_content = file.read()
except Exception as e:
self.reporter.report_failure(path,
f"failed to load view file: {e}")
return None
try:
with open(path, "r") as file:
logger.debug(f"Loading viewfile {path}")
parsed = lkml.load(file)
looker_viewfile = LookerViewFile.from_looker_dict(
absolute_file_path=path,
looker_view_file_dict=parsed,
base_folder=self._base_folder,
raw_file_content=raw_file_content,
reporter=reporter,
)
logger.debug(f"adding viewfile for path {path} to the cache")
self.viewfile_cache[path] = looker_viewfile
return looker_viewfile
except Exception as e:
self.reporter.report_failure(path,
f"failed to load view file: {e}")
return None
def main():
kafka_producer = make_kafka_producer(EXTRA_KAFKA_CONF)
viewfile_loader = LookerViewFileLoader()
looker_models = []
all_views = []
model_files = sorted(f for f in glob.glob(
f"{LOOKER_DIRECTORY}/**/*.model.lkml", recursive=True))
for f in model_files:
try:
with open(f, 'r') as file:
parsed = lkml.load(file)
looker_model = LookerModel.from_looker_dict(parsed)
looker_models.append(looker_model)
except Exception as e:
print(e)
print(f"Error processing model file {f}. Skipping it")
for model in looker_models:
for include in model.resolved_includes:
looker_viewfile = viewfile_loader.load_viewfile(
include, model.connection)
if looker_viewfile is not None:
for raw_view in looker_viewfile.views:
maybe_looker_view = LookerView.from_looker_dict(
raw_view, model.connection, looker_viewfile,
viewfile_loader)
if maybe_looker_view:
all_views.append(maybe_looker_view)
for view in all_views:
MCE = build_dataset_mce(view)
print(view)
print(MCE)
kafka_producer.produce(topic=KAFKA_TOPIC,
key=MCE['proposedSnapshot'][1]['urn'],
value=MCE)
kafka_producer.flush()
def test_load_with_bad_argument_raises_type_error():
with pytest.raises(TypeError):
lkml.load(stream=100)
def load(filename):
"""Helper method to load a LookML file from tests/resources and parse it."""
path = Path(__file__).parent / "resources" / filename
with path.open() as file:
return lkml.load(file)