def _build_dependencies_query(cls,
team_id,
topic,
label,
node,
filter_on_config=False):
"""Build a Cypher query based on given parameters."""
where = ""
query = ("MATCH(n:{topic}_{label}{{name: '{name}'}}) "
"OPTIONAL MATCH (n)-[r]->(m) {where}"
"RETURN n,r,m ORDER BY m.name LIMIT 10")
# Filter the dependencies using the labels declared in the configuration
if filter_on_config:
label_config = ConfigController.get_label_config(team_id, label)
regex = r"^.*(\[[A-Za-z]+(, [A-Za-z]+)*?\])$"
match = re.search(regex, label_config["qos"])
if match:
deps = match.group(1)[1:-1].split(", ")
where += "WHERE '{}_{}' IN LABELS(m) ".format(topic, deps[0])
for dep in deps[1:]:
where += "OR '{}_{}' IN LABELS(m) ".format(topic, dep)
return query.format(where=where, topic=topic, label=label, name=node)
def get_current_config(team_id):
"""
.. :quickref: GET; Lorem ipsum."""
if not TeamPermission.is_user(team_id):
abort(403)
config = ConfigController.get_current_config(team_id=team_id)
return jsonify(config), 200
def get_teams_schema(ds, **kwargs):
"""
This task lists the last config for every team. Then a Neo4j
query is done to count the nodes of each label.
"""
with kwargs["params"]["app"].app_context():
from depc.controllers.configs import ConfigController
# Get all configs ordered by -date
configs = ConfigController._list(order_by="updated_at", reverse=True)
# Get the last config by team
teams = {}
for config in configs:
team = config.team
# For each team
if team.kafka_topic not in teams.keys():
logger.info("[{0}] Configuration : {1}".format(team.name, config.data))
data = {
"id": str(team.id),
"name": team.name,
"topic": team.kafka_topic,
"schema": config.data,
"labels": {},
}
# Count number of nodes per label
logger.info(
"[{0}] Counting nodes for {1} labels...".format(
team.name, len(config.data.keys())
)
)
for label in config.data.keys():
neo_key = "{}_{}".format(team.kafka_topic, label)
records = get_records(
"MATCH (n:{label}) RETURN count(n) AS Count".format(
label=neo_key
)
)
count = list(records)[0].get("Count")
logger.info(
"[{0}] {1} nodes for label {2}...".format(
team.name, count, label
)
)
data["labels"][label] = count
teams[team.kafka_topic] = data
# Save the config into an Airflow variable
Variable.set("config", list(teams.values()), serialize_json=True)
def get_labels(cls, team_id):
"""
This function returns the list of labels for a team. The labels can be
found in 2 places : in the configuration or directly in Neo4j.
Having a label in Neo4j does not mean it's present in the config, and
having a label in the config does not mean it has been created in Neo4j.
"""
team = TeamController._get({"Team": {"id": team_id}})
# Get labels per team
neo = Neo4jClient()
query = (
"CALL db.labels() YIELD label "
"WITH split(label, '_')[0] AS Team, split(label, '_')[1] AS Label "
"RETURN Team, collect(Label) AS Labels "
"ORDER BY Team")
results = neo.query(query, returns=(str, list))
# Get the label for the current team
labels_per_teams = {r[0]: r[1] for r in results}
# Pick the wanted team's labels
team_labels = labels_per_teams.get(team.kafka_topic, [])
# Get the number of nodes per label
labels = {}
if team_labels:
query = cls._build_query_count_nodes(team.kafka_topic, team_labels)
results = neo.query(query, returns=(str, int))
# Merge the Neo4j and DB labels
labels = {
r[0]: {
"name": r[0],
"nodes_count": r[1],
"qos_query": None
}
for r in results
}
# No config: return the list of labels
try:
config = ConfigController.get_current_config(team_id)
except NotFoundError:
return list(labels.values())
# Add the QoS query for each label
qos_queries = {l: d["qos"] for l, d in config["data"].items()}
for label, query in qos_queries.items():
if label not in labels:
labels[label] = {"name": label, "nodes_count": 0}
labels[label]["qos_query"] = query
return list(labels.values())
def revert_config(team_id, config_id):
"""
.. :quickref: PUT; Lorem ipsum."""
if not TeamPermission.is_manager(team_id):
abort(403)
config = ConfigController.revert_config(team_id=team_id,
config_id=config_id)
if not config:
abort(404)
return jsonify(config), 200
def export_summary_grafana(cls, team):
data = cls._generate_grafana_template(team,
"grafana_summary_dashboard")
try:
configs = ConfigController.get_current_config(team.id)["data"]
data = data.replace("$$MERMAIDDIAGRAM$$",
cls._generate_marmaid_diagram(configs))
except NotFoundError:
data = data.replace("$$MERMAIDDIAGRAM$$", "")
return json.loads(data)
def list_configs(team_id):
"""
.. :quickref: GET; Lorem ipsum."""
if not TeamPermission.is_user(team_id):
abort(403)
configs = ConfigController.list(filters={"Config": {
"team_id": team_id
}},
order_by="updated_at",
reverse=True)
return jsonify(configs), 200
def put_config(team_id):
"""
.. :quickref: POST; Lorem ipsum."""
if not TeamPermission.is_manager(team_id):
abort(403)
payload = request.get_json(force=True)
current_conf = ConfigController.create({
"team_id": team_id,
"data": payload
})
return jsonify(current_conf), 200
def get_config(team_id, config_id):
"""
.. :quickref: GET; Lorem ipsum."""
if not TeamPermission.is_user(team_id):
abort(403)
config = ConfigController.get(
filters={"Config": {
"team_id": team_id,
"id": config_id
}})
if not config:
abort(404)
return jsonify(config), 200
def _create_config(team_id, conf={}):
with app.app_context():
return ConfigController.create({
'team_id': team_id,
'data': conf
})
def get_labels(cls, team_id):
"""
This function returns the list of labels for a team. The labels can be
found in 2 places : in the configuration or directly in Neo4j.
Having a label in Neo4j does not mean it's present in the config, and
having a label in the config does not mean it has been created in Neo4j.
"""
team = TeamController._get({"Team": {"id": team_id}})
# Get labels per team
neo = Neo4jClient()
query = """CALL db.labels() YIELD label
WITH split(label, '_')[0] AS Team, split(label, '_')[1] AS Label
RETURN Team, collect(Label) AS Labels
ORDER BY Team"""
results = neo.query(query, returns=(str, list))
# Get the label for the current team
labels_per_teams = {r[0]: r[1] for r in results}
# Pick the wanted team's labels
team_labels = labels_per_teams.get(team.kafka_topic, [])
# Construct the cypher to get the number of nodes per label
# TODO : add cache for this feature
labels = {}
if team_labels:
query = ("MATCH (n:{team}_{label}) "
'WITH "{label}" AS Label, count(n) AS Count '
"RETURN Label, Count")
query = query.format(team=team.kafka_topic, label=team_labels[0])
for label in team_labels[1:]:
union = ("\nUNION MATCH (n:{team}_{label}) "
'WITH "{label}" AS Label, count(n) AS Count '
"RETURN Label, Count")
query += union.format(team=team.kafka_topic, label=label)
results = neo.query(query, returns=(str, int))
# Merge the Neo4j and DB labels
labels = {
r[0]: {
"name": r[0],
"nodes_count": r[1],
"qos_query": None
}
for r in results
}
try:
config = ConfigController.get_current_config(team_id)
except NotFoundError:
return list(labels.values())
qos_queries = {l: d["qos"] for l, d in config["data"].items()}
for label, query in qos_queries.items():
if label not in labels:
labels[label] = {"name": label, "nodes_count": 0}
labels[label]["qos_query"] = query
return list(labels.values())
def get_node_dependencies(cls,
team_id,
label,
node,
filter_on_config=False):
team = TeamController._get({"Team": {"id": team_id}})
query = """
MATCH(n:{topic}_{label}{{name: "{name}"}})
OPTIONAL MATCH (n)-[r]->(m) {where}
RETURN n,r,m
ORDER BY m.name LIMIT 10
"""
# Filter the dependencies with the labels declared in the configuration
where = ""
if filter_on_config:
label_config = ConfigController.get_label_config(team_id, label)
regex = r"^.*(\[[A-Za-z]+(, [A-Za-z]+)*?\])$"
match = re.search(regex, label_config["qos"])
if match:
deps = match.group(1)[1:-1].split(", ")
where += "WHERE '{}_{}' IN LABELS(m)".format(
team.kafka_topic, deps[0])
for dep in deps[1:]:
where += " OR '{}_{}' IN LABELS(m)".format(
team.kafka_topic, dep)
neo = Neo4jClient()
query = query.format(topic=team.kafka_topic,
label=label,
name=node,
where=where)
sequence = neo.query(query, data_contents=DATA_GRAPH)
dependencies = {
"dependencies": {},
"graph": {
"nodes": [],
"relationships": []
}
}
if sequence.graph:
for g in sequence.graph:
# Handle the nodes
for node in g["nodes"]:
title = node["labels"][0][len(team.kafka_topic) + 1:]
if title not in dependencies["dependencies"]:
dependencies["dependencies"][title] = []
# Id is required to make data unique later
node["properties"]["id"] = node["id"]
dependencies["dependencies"][title].append(
node["properties"])
# Add the node data in the whole graph
dependencies["graph"]["nodes"].append({
"id":
node["id"],
"label":
node["properties"].get("name", "unknown"),
"title":
title,
})
# Handle the relationships
for rel in g["relationships"]:
dependencies["graph"]["relationships"].append({
"id":
rel["id"],
"from":
rel["startNode"],
"to":
rel["endNode"],
"arrows":
"to",
})
dependencies = cls.make_unique(dependencies)
return dependencies