def test_validation_success():
m = Manager({'graphs': [{'nodes': [{'id': 'node_id', 'module': 'foobar', 'class': 'Foobar'}]}]})
assert m != None
m = Manager({'graphs': [{'nodes': [{'id': 'node_id', 'module': 'foobar', 'class': 'Foobar'}], 'edges': [{'source': 'node_id', 'target': 'node_id'}]}]})
assert m != None
m = Manager({'graphs': [{'nodes': [{'id': 'node_id', 'module': 'foobar', 'class': 'Foobar'}], 'edges': [{'source': 'node:*', 'target': 'node_id'}]}]})
assert m != None
m = Manager({'graphs': [{'nodes': [{'id': 'node_id', 'module': 'foobar', 'class': 'Foobar'}], 'edges': [{'source': 'node:1', 'target': 'node:abc'}]}]})
assert m != None
def test_validation_failure():
with pytest.raises(ValueError):
m = Manager({'graphs!': [{'nodes': [{'id': 'node_id', 'module': 'foobar', 'class': 'Foobar'}]}]})
with pytest.raises(ValueError):
m = Manager({'graphs': [{'nodes': [{'id': 'node_id', 'module': '.foobar', 'class': 'Foobar'}]}]})
with pytest.raises(ValueError):
m = Manager({'graphs': [{'nodes': [{'id': 'node_id', 'module': 'foobar.', 'class': 'Foobar'}]}]})
with pytest.raises(ValueError):
m = Manager({'graphs': [{'nodes': [{'id': 'node_id', 'module': 'foobar.', 'class': 'foobar'}]}]})
with pytest.raises(ValueError):
m = Manager({'graphs': [{'nodes': [{'id': 'node!', 'module': 'foobar.', 'class': 'foobar'}]}]})
def main():
sys.path.append(os.getcwd())
args = _args()
load_dotenv(args.env)
_init_logging(args.debug)
LOGGER.info("Timeflux %s" % __version__)
_run_hook("pre")
try:
Manager(args.app).run()
except Exception as error:
LOGGER.error(error)
_terminate()
def yaml_to_png(filename, format="png", sort=False):
"""Generate an image from a YAML application file.
Args:
filename (string): The path to the YAML application file.
format (string): The image format. Default: `png`.
sort (boolean): If `True`, the graphs will be sorted in the same topological order
that is used to run the application. Default: `False`.
"""
# Load graphs
graphs = Manager(filename)._graphs
# Default graph attributes
graph_attr = {
"splines": "spline",
"rankdir": "LR",
"style": "filled",
"fillcolor": "lightgrey",
"color": "black",
"fontname": "helvetica",
"fontsize": "16",
}
# Default node attributes
node_attr = {
"shape": "ellipse",
"style": "filled",
"fillcolor": "white",
"color": "black",
"fontname": "helvetica",
"fontsize": "14",
}
# Default edge attributes
edge_attr = {
"fontname": "helvetica",
"fontsize": "9",
"fontcolor": "snow4",
"labeldistance": "1.5",
"labelangle": "-25",
}
# Special nodes
broker = None
publishers = []
subscribers = []
# Initialize the viz using the default dot engine
dot = gv.Digraph(format=format,
graph_attr=graph_attr,
node_attr=node_attr,
edge_attr=edge_attr)
# Add clusters
for ci, graph in enumerate(graphs):
cluster_name = f"cluster_{ci}"
with dot.subgraph(name=cluster_name) as cluster:
# Set label
if "id" in graph:
cluster.attr(label=graph["id"])
# Add nodes
nodes = {}
for ni, node in enumerate(graph["nodes"]):
node_name = f"{cluster_name}_node_{ni}"
nodes[node["id"]] = node_name
cluster.node(node_name, label=node["id"])
# Check for special nodes
if node["module"] == "timeflux.nodes.zmq":
if node["class"] in ("Broker", "BrokerMonitored",
"BrokerLVC"):
broker = node_name
elif node["class"] == "Pub":
publishers.append({
"name": node_name,
"topic": node["params"]["topic"]
})
elif node["class"] == "Sub":
subscribers.append({
"name": node_name,
"topics": node["params"]["topics"]
})
# Add edges
if "edges" in graph:
for edge in graph["edges"]:
src = edge["source"].split(":")
dst = edge["target"].split(":")
taillabel = src[1] if len(src) == 2 else ""
headlabel = dst[1] if len(dst) == 2 else ""
cluster.edge(
nodes[src[0]],
nodes[dst[0]],
taillabel=taillabel,
headlabel=headlabel,
)
# Sort nodes according to the topoligical order of the graph
# This is still a bit hacky and should be considered as a WIP.
if sort:
path = Graph(graph).traverse()
edge = []
for node in path:
edge.append(nodes[node["node"]])
if len(edge) == 2:
cluster.edge(edge[0],
edge[1],
style="invis",
weight="100")
edge = [edge[1]]
# Add special edges
if broker:
dot.attr("edge", style="dashed")
for publisher in publishers:
dot.edge(publisher["name"], broker, label=publisher["topic"])
for subscriber in subscribers:
for topic in subscriber["topics"]:
dot.edge(broker, subscriber["name"], label=topic)
print(dot.source)
out = os.path.splitext(os.path.basename(filename))[0]
dot.render(out, cleanup=True, view=True)
def yaml_to_png(filename, format='png', sort=False):
"""Generate an image from a YAML application file.
Args:
filename (string): The path to the YAML application file.
format (string): The image format. Default: `png`.
sort (boolean): If `True`, the graphs will be sorted in the same topological order
that is used to run the application. Default: `False`.
"""
# Load graphs
graphs = Manager(filename)._graphs
# Default graph attributes
graph_attr = {
'splines': 'spline',
'rankdir': 'LR',
'style': 'filled',
'fillcolor': 'lightgrey',
'color': 'black',
'fontname': 'helvetica',
'fontsize': '16'
}
# Default node attributes
node_attr = {
'shape': 'ellipse',
'style': 'filled',
'fillcolor': 'white',
'color': 'black',
'fontname': 'helvetica',
'fontsize': '14'
}
# Default edge attributes
edge_attr = {
'fontname': 'helvetica',
'fontsize': '9',
'fontcolor': 'snow4',
'labeldistance': '1.5',
'labelangle': '-25'
}
# Special nodes
broker = None
publishers = []
subscribers = []
# Initialize the viz using the default dot engine
dot = gv.Digraph(format=format, graph_attr=graph_attr, node_attr=node_attr, edge_attr=edge_attr)
# Add clusters
for ci, graph in enumerate(graphs):
cluster_name = f'cluster_{ci}'
with dot.subgraph(name=cluster_name) as cluster:
# Set label
if 'id' in graph:
cluster.attr(label=graph['id'])
# Add nodes
nodes = {}
for ni, node in enumerate(graph['nodes']):
node_name = f'{cluster_name}_node_{ni}'
nodes[node['id']] = node_name
cluster.node(node_name, label=node['id'])
# Check for special nodes
if node['module'] == 'timeflux.nodes.zmq':
if node['class'] in ('Broker', 'BrokerMonitored', 'BrokerLVC'):
broker = node_name
elif node['class'] == 'Pub':
publishers.append({'name': node_name, 'topic': node['params']['topic']})
elif node['class'] == 'Sub':
subscribers.append({'name': node_name, 'topics': node['params']['topics']})
# Add edges
if 'edges' in graph:
for edge in graph['edges']:
src = edge['source'].split(':')
dst = edge['target'].split(':')
taillabel = src[1] if len(src) == 2 else ''
headlabel = dst[1] if len(dst) == 2 else ''
cluster.edge(nodes[src[0]], nodes[dst[0]], taillabel=taillabel, headlabel=headlabel)
# Sort nodes according to the topoligical order of the graph
# This is still a bit hacky and should be considered as a WIP.
if sort:
path = Graph(graph).traverse()
edge = []
for node in path:
edge.append(nodes[node['node']])
if len(edge) == 2:
cluster.edge(edge[0], edge[1], style='invis', weight='100')
edge = [edge[1]]
# Add special edges
if broker:
dot.attr('edge', style='dashed')
for publisher in publishers:
dot.edge(publisher['name'], broker, label=publisher['topic'])
for subscriber in subscribers:
for topic in subscriber['topics']:
dot.edge(broker, subscriber['name'], label=topic)
print(dot.source)
out = os.path.splitext(os.path.basename(filename))[0]
dot.render(out, cleanup=True, view=True)
def test_import_recursive():
m = Manager(pytest.path + '/graphs/import3.yaml')
assert len(m._imports) == 5
def test_import():
m = Manager(pytest.path + '/graphs/import.yaml')
assert len(m._imports) == 4
def test_load_dict():
m = Manager(test_config)
assert m._graphs == test_config['graphs']
def test_load_json_file():
m = Manager(pytest.path + '/graphs/test.json')
assert m._graphs == test_config['graphs']
def test_load_yaml_file():
m = Manager(pytest.path + '/graphs/test.yaml')
assert m._graphs == test_config['graphs']
def test_load_invalid_file():
with pytest.raises(FileNotFoundError):
Manager('foo.yaml')
def test_load_invalid_config():
with pytest.raises(ValueError):
Manager(42)
def test_template():
os.environ['FOOBAR'] = 'MyClass'
m = Manager(pytest.path + '/graphs/template.yaml')
assert m._graphs[0]['nodes'][0]['class'] == 'MyClass'
