def test_process_file5(config):
grapher = LookMlGrapher(config)
with pytest.raises(Exception) as e:
lookml = LookML("test/empty.view.lkml")
grapher.process_lookml(lookml)
assert 'No models, views, or explores? test/empty.view.lkml' in str(
e.value)
def test_process_file3(config):
grapher = LookMlGrapher(config)
assert grapher.node_map == {}
lookml = LookML("test/grapher_lkml/some_view.view.lkml")
grapher.process_lookml(lookml)
assert 'some_view' in grapher.node_map
assert grapher.node_map['some_view'] == NodeType.VIEW
def test_process_file(config):
grapher = LookMlGrapher(config)
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
grapher.process_lookml(LookML("test/grapher_lkml/some_model.model.lkml"))
assert grapher.models_to_explores == [('some_model', 'some_explore')]
assert grapher.explores_to_views == [('some_explore', 'some_view'),
('some_explore', 'some_other_view')]
def test_process_file3(config):
grapher = LookMlGrapher(config)
assert grapher.node_map == {}
grapher.process_file("test/grapher_lkml/some_view.view.lkml")
assert 'some_view' in grapher.node_map
assert grapher.node_map['some_view'] == NodeType.VIEW
if os.path.exists(config['tmp_file']):
os.remove(config['tmp_file'])
def test_process_file(config):
grapher = LookMlGrapher(config)
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
grapher.process_file("test/grapher_lkml/some_model.model.lkml")
assert grapher.models_to_explores == [('some_model', 'some_explore')]
assert grapher.explores_to_views == [('some_explore', 'some_view'),
('some_explore', 'some_other_view')]
if os.path.exists(config['tmp_file']):
os.remove(config['tmp_file'])
def __init__(self, config):
"""Initialize the rule
Args:
config (JSON): configuration
"""
self.config = config
self.grapher = LookMlGrapher(config)
self.view_dict = {}
def test_process_file4(config):
grapher = LookMlGrapher(config)
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
lookml = LookML("test/grapher_lkml/some_explore.explore.lkml")
grapher.process_lookml(lookml)
assert grapher.models_to_explores == []
assert grapher.explores_to_views == [
("some_explore", "some_view"),
("some_explore", "some_other_view"),
]
def test_run(config):
grapher = LookMlGrapher(config)
img_file = config['output']
if os.path.exists(config['output']):
os.remove(img_file)
grapher.run()
assert os.path.exists(img_file)
if os.path.exists(config['output']):
os.remove(img_file)
def test_process_file2(config):
grapher = LookMlGrapher(config)
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
json_data = grapher.lookml.get_json_representation(
"test/grapher_lkml/some_model.model.lkml")
grapher.process_file(None, json_data)
assert grapher.models_to_explores == [('some_model', 'some_explore')]
assert grapher.explores_to_views == [('some_explore', 'some_view'),
('some_explore', 'some_other_view')]
if os.path.exists(config['tmp_file']):
os.remove(config['tmp_file'])
def test_process_explores(config):
grapher = LookMlGrapher(config)
lookml = LookML("test/grapher_lkml/some_model.model.lkml")
m = lookml.base_name
e = lookml.json_data['explores'][0]
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
grapher.process_explores(m, e)
assert grapher.models_to_explores == [('some_model', 'some_explore')]
assert grapher.explores_to_views == [('some_explore', 'some_view'),
('some_explore', 'some_other_view')]
def test_orphans(config):
grapher = LookMlGrapher(config)
grapher.node_map['model_a'] = NodeType.MODEL
grapher.node_map['explore_a'] = NodeType.EXPLORE
grapher.node_map['view_a'] = NodeType.VIEW
grapher.node_map['orphan'] = NodeType.VIEW
grapher.models_to_explores.append(('model_a', 'explore_a'))
grapher.explores_to_views.append(('explore_a', 'view_a'))
grapher.tag_orphans()
orphans = grapher.orphans()
assert len(orphans) == 1
assert list(orphans)[0] == 'orphan'
def test_orphans(config):
grapher = LookMlGrapher(config)
grapher.node_map["model_a"] = NodeType.MODEL
grapher.node_map["explore_a"] = NodeType.EXPLORE
grapher.node_map["view_a"] = NodeType.VIEW
grapher.node_map["orphan"] = NodeType.VIEW
grapher.models_to_explores.append(("model_a", "explore_a"))
grapher.explores_to_views.append(("explore_a", "view_a"))
grapher.tag_orphans()
orphans = grapher.orphans()
assert len(orphans) == 1
assert list(orphans)[0] == "orphan"
def test_extract_graph_info(config):
grapher = LookMlGrapher(config)
assert grapher.node_map == {}
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
grapher.extract_graph_info(["test/grapher_lkml/some_model.model.lkml"])
assert grapher.models_to_explores == [('some_model', 'some_explore')]
assert grapher.explores_to_views == [('some_explore', 'some_view'),
('some_explore', 'some_other_view')]
assert len(grapher.node_map) == 2
assert 'some_explore' in grapher.node_map
assert grapher.node_map['some_explore'] == NodeType.EXPLORE
assert 'some_model' in grapher.node_map
assert grapher.node_map['some_model'] == NodeType.MODEL
def run(
lkml_paths: list[str],
output: str = "lookml_diagram.png",
title: str = "Hello, World!",
**kwargs,
):
# create the LookML grapher
grapher = LookMlGrapher(config="None")
# parse LookML and extract graph info from the input files
grapher.extract_graph_info(lkml_paths)
# create nx DiGraph object
g = grapher.create_graph()
return grapher, g
def grapher(args):
if args.image_height:
config["grapher"]["options"]["image_height"] = args.image_height
if args.image_width:
config["grapher"]["options"]["image_width"] = args.image_width
if args.node_size:
config["grapher"]["options"]["node_size"] = args.node_size
LookMlGrapher().run()
def test_process_explores(config):
grapher = LookMlGrapher(config)
lookml = LookML("test/grapher_lkml/some_model.model.lkml")
m = lookml.base_name
e = lookml.json_data["explores"][0]
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
grapher.process_explores(m, e)
assert grapher.models_to_explores == [("some_model", "some_explore")]
assert grapher.explores_to_views == [
("some_explore", "some_view"),
("some_explore", "some_other_view"),
]
def test_extract_graph_info(config):
grapher = LookMlGrapher(config)
assert grapher.node_map == {}
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
grapher.extract_graph_info(["test/grapher_lkml/some_model.model.lkml"])
assert grapher.models_to_explores == [("some_model", "some_explore")]
assert grapher.explores_to_views == [
("some_explore", "some_view"),
("some_explore", "some_other_view"),
]
assert len(grapher.node_map) == 2
assert "some_explore" in grapher.node_map
assert grapher.node_map["some_explore"] == NodeType.EXPLORE
assert "some_model" in grapher.node_map
assert grapher.node_map["some_model"] == NodeType.MODEL
def test_process_explores(config):
grapher = LookMlGrapher(config)
json_data = grapher.lookml.get_json_representation(
"test/grapher_lkml/some_model.model.lkml")
m = json_data['files'][0]['models'][0]
e = m['explores'][0]
assert grapher.models_to_explores == []
assert grapher.explores_to_views == []
grapher.process_explores(m, e)
assert grapher.models_to_explores == [('some_model', 'some_explore')]
assert grapher.explores_to_views == [('some_explore', 'some_view'),
('some_explore', 'some_other_view')]
if os.path.exists(config['tmp_file']):
os.remove(config['tmp_file'])
def main():
"""
"""
logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s %(funcName)s: %(message)s', level=logging.INFO)
args, _ = parse_arguments()
if os.path.exists(args.config):
with open(args.config, 'r') as f:
config = json.load(f)
else:
raise Exception('config file at: {} not found'.format(args.config))
LookMlGrapher(config).run()
def test_create_graph(config):
grapher = LookMlGrapher(config)
grapher.node_map['model_a'] = NodeType.MODEL
grapher.node_map['explore_a'] = NodeType.EXPLORE
grapher.node_map['view_a'] = NodeType.VIEW
grapher.models_to_explores.append(('model_a', 'explore_a'))
grapher.explores_to_views.append(('explore_a', 'view_a'))
g = grapher.create_graph()
assert isinstance(g, nx.DiGraph)
assert len(g) == 3
def generate_images(self, repo, commits, directory):
"""given a set of commits, run the LookML grapher to produce one image per commit
Args:
commits (list): list of commits in going forward in time (oldest -> newest)
directory (str): directory to save the image files to
Returns:
nothing. Side effect is to create a set of images in a directory
"""
filenames = []
for i, commit in enumerate(commits):
commit_id = str(commit)
ts = int(commit.committed_date)
dt = datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
logging.info("Processing %d %s %s" % (i, commit.message, dt))
repo.git.checkout(commit_id)
# add some metadata to the config...
config = self.config
filename = directory + os.path.sep + "img" + str(i) + ".png"
config["output"] = filename
config["options"]["title"] = dt
filenames.append(filename)
try:
grapher = LookMlGrapher(config).run()
except Exception as e:
print("issue with " + str(i) + "," + commit_id)
# raise e
return filenames
class NoOrphansRule(Rule):
'''
Look for views unreferenced by any explores
'''
def __init__(self, config):
'''Initialize the rule
Args:
config (JSON): configuration
'''
self.config = config
self.grapher = LookMlGrapher(config)
self.view_dict = {}
def run(self, json_data):
'''run the rule
Args:
json_data (JSON): json_data of the lookml-parser dictionary for this file
Returns:
(tuple): tuple containing:
relevant (bool): is this rule relevant for this JSON chunk?
passed (bool): did the rule pass?
'''
pass # pragma: no cover
def process_file(self, json_data):
'''process the JSON_DATA of a file, delegating down to the grapher
also store metadata we'll need in a later stage
Args:
json_data (JSON): json_data: parsed lkml file as JSON
Returns:
nothing. side effect is to store data in grapher and in this class
'''
self.grapher.process_file(filepath=None, json_data=json_data)
# we'll need the view_namme->filename mapping to output later
if 'views' in json_data['files'][0]:
view_name = json_data['files'][0]['views'][0]['_view']
filepath = json_data['files'][0]['_file_path']
self.view_dict[view_name] = filepath
def finish_up(self, file_out):
'''find the orphans, if any, and add results to file_out
Args:
file_out (list): list of results for files
Returns:
file_out (list)
'''
self.grapher.tag_orphans()
orphans = self.grapher.orphans()
for orphan in orphans:
simple_filepath = self.view_dict[orphan] #['files'][0]['_file_path']
logging.info("Found orphan %s in %s", orphan, simple_filepath)
out = {"file": simple_filepath, "rule": self.name(), "passed": 0}
file_out.append(out)
return (file_out)
def test_plot_graph(config):
img_file = config["output"]
if os.path.exists(config["output"]):
os.remove(img_file)
grapher = LookMlGrapher(config)
grapher.node_map["model_a"] = NodeType.MODEL
grapher.node_map["explore_a"] = NodeType.EXPLORE
grapher.node_map["view_a"] = NodeType.VIEW
grapher.node_map["orphan"] = NodeType.VIEW
grapher.models_to_explores.append(("model_a", "explore_a"))
grapher.explores_to_views.append(("explore_a", "view_a"))
grapher.tag_orphans()
g = grapher.create_graph()
grapher.plot_graph(g, img_file, "some title")
assert os.path.exists(img_file)
if os.path.exists(config["output"]):
os.remove(img_file)
class NoOrphansRule(Rule):
"""
Look for views unreferenced by any explores
"""
def __init__(self, config):
"""Initialize the rule
Args:
config (JSON): configuration
"""
self.config = config
self.grapher = LookMlGrapher(config)
self.view_dict = {}
def run(self, json_data):
"""run the rule
Args:
json_data (JSON): json_data of the lkml-parsed JSON dictionary for this file
Returns:
(tuple): tuple containing:
relevant (bool): is this rule relevant for this JSON chunk?
passed (bool): did the rule pass?
"""
pass # pragma: no cover
def process_lookml(self, lookml):
"""process the JSON_DATA of a file, delegating down to the grapher
also store metadata we'll need in a later stage
Args:
lookmlk (LookML): instance of LookML
Returns:
nothing. side effect is to store data in grapher and in this class
"""
self.grapher.process_lookml(lookml)
# we'll need the view_namme->filename mapping to output later
if lookml.has_views():
v = lookml.views()[0]
view_name = lookml.base_name
filepath = lookml.infilepath
self.view_dict[view_name] = filepath
def finish_up(self, file_out):
"""find the orphans, if any, and add results to file_out
Args:
file_out (list): list of results for files
Returns:
file_out (list)
"""
self.grapher.tag_orphans()
orphans = self.grapher.orphans()
for orphan in orphans:
simple_filepath = self.view_dict[orphan]
logging.info("Found orphan %s in %s", orphan, simple_filepath)
out = {"file": simple_filepath, "rule": self.name(), "passed": 0}
file_out.append(out)
return file_out
def test_plot_graph(config):
img_file = config['output']
if os.path.exists(config['output']):
os.remove(img_file)
grapher = LookMlGrapher(config)
grapher.node_map['model_a'] = NodeType.MODEL
grapher.node_map['explore_a'] = NodeType.EXPLORE
grapher.node_map['view_a'] = NodeType.VIEW
grapher.node_map['orphan'] = NodeType.VIEW
grapher.models_to_explores.append(('model_a', 'explore_a'))
grapher.explores_to_views.append(('explore_a', 'view_a'))
grapher.tag_orphans()
g = grapher.create_graph()
grapher.plot_graph(g, img_file, "some title")
assert os.path.exists(img_file)
if os.path.exists(config['output']):
os.remove(img_file)
