def get_label_nodes(cls,
team_id,
label,
name=None,
limit=None,
random=False):
team = TeamController._get({"Team": {"id": team_id}})
neo = Neo4jClient()
query = "MATCH (n:{}_{}) WITH n".format(team.kafka_topic, label)
if random:
query += ", rand() as r ORDER BY r"
if name:
query += " WHERE n.name CONTAINS '{0}'".format(name)
query += " RETURN distinct(n.name)"
try:
limit = int(limit)
query += " LIMIT {0}".format(limit)
except (TypeError, ValueError):
pass
results = neo.query(query, returns=(str, ))
return [r[0] for r in results]
def delete_node(cls, team_id, label, node, detach=False):
team = TeamController._get({"Team": {"id": team_id}})
neo = Neo4jClient()
query = 'MATCH(n:{topic}_{label}{{name: "{name}"}})'.format(
topic=team.kafka_topic, label=label, name=node)
# First we check if the node has dependencies
has_deps = query + "-[r:DEPENDS_ON]->() RETURN count(r)"
count = neo.query(has_deps,
returns=(int, ))[0][0] # return is weird : [[100]]
if count and not detach:
msg = (
"Can not delete node {node} because it still has {count} relationships"
)
raise RequirementsNotSatisfiedError(
msg.format(node=node, count=count))
# We can delete the node and its relationships
query += " DETACH DELETE n"
try:
# Neo4j returns nothing when deleting a node
neo.query(query)
except TransactionException as e:
raise RequirementsNotSatisfiedError(str(e))
return {}
def count_node_dependencies(cls, team_id, label, node):
team = TeamController._get({"Team": {"id": team_id}})
neo = Neo4jClient()
query = ("MATCH(n:{0}_{1}{{name: '{2}'}}) "
"OPTIONAL MATCH (n)-[:DEPENDS_ON]->(m) "
"RETURN count(m)").format(team.kafka_topic, label, node)
sequence = neo.query(query, data_contents=DATA_GRAPH)
return {"count": sequence[0][0]}
def get_labels(cls, team_id):
"""
This function returns the list of labels for a team. The labels can be
found in 2 places : in the configuration or directly in Neo4j.
Having a label in Neo4j does not mean it's present in the config, and
having a label in the config does not mean it has been created in Neo4j.
"""
team = TeamController._get({"Team": {"id": team_id}})
# Get labels per team
neo = Neo4jClient()
query = (
"CALL db.labels() YIELD label "
"WITH split(label, '_')[0] AS Team, split(label, '_')[1] AS Label "
"RETURN Team, collect(Label) AS Labels "
"ORDER BY Team")
results = neo.query(query, returns=(str, list))
# Get the label for the current team
labels_per_teams = {r[0]: r[1] for r in results}
# Pick the wanted team's labels
team_labels = labels_per_teams.get(team.kafka_topic, [])
# Get the number of nodes per label
labels = {}
if team_labels:
query = cls._build_query_count_nodes(team.kafka_topic, team_labels)
results = neo.query(query, returns=(str, int))
# Merge the Neo4j and DB labels
labels = {
r[0]: {
"name": r[0],
"nodes_count": r[1],
"qos_query": None
}
for r in results
}
# No config: return the list of labels
try:
config = ConfigController.get_current_config(team_id)
except NotFoundError:
return list(labels.values())
# Add the QoS query for each label
qos_queries = {l: d["qos"] for l, d in config["data"].items()}
for label, query in qos_queries.items():
if label not in labels:
labels[label] = {"name": label, "nodes_count": 0}
labels[label]["qos_query"] = query
return list(labels.values())
def _get_team(cls, team_id):
obj = TeamController._get(filters={"Team": {"id": team_id}})
# Add the members of the team
team = TeamController.resource_to_dict(obj)
team.update({
"members": [m.name for m in obj.members],
"editors": [m.name for m in obj.editors],
"managers": [m.name for m in obj.managers],
})
return team
def get_impacted_nodes_all(cls, team_id, label, node, impacted_label, ts,
with_inactive_nodes):
try:
ts = int(ts)
except (TypeError, ValueError):
raise IntegrityError("'ts' parameter must be a positive number")
if ts < 0:
raise IntegrityError("'ts' parameter must be a positive number")
if not impacted_label:
raise IntegrityError("'impactedLabel' parameter must not be empty")
team = TeamController._get({"Team": {"id": team_id}})
json_string = "["
nodes_batch = 50000
skip = 0
total_count = cls.get_impacted_nodes_count(team_id, label, node,
impacted_label)["count"]
# Load all impacted nodes data by batch inside a list
while skip < total_count:
query = cls._build_impacted_nodes_queries(
topic=team.kafka_topic,
label=label,
node=node,
impacted_label=impacted_label,
skip=skip,
limit=nodes_batch,
count=False,
)
new_json_string_data = json.dumps(
cls._compute_impacted_nodes_from_data(
get_records(query).data(),
ts,
with_inactive_nodes=with_inactive_nodes,
),
indent=4,
)
skip += nodes_batch
# if this is not the last loop, add a "," to the last element
# of the array to help easy concatenation at the next loop
if skip < total_count:
new_json_string_data = new_json_string_data[:-2] + ",\n]"
json_string += new_json_string_data[1:-2]
json_string += "\n]"
return json_string
def get_label_nodes(cls,
team_id,
label,
name=None,
limit=None,
random=False):
team = TeamController._get({"Team": {"id": team_id}})
neo = Neo4jClient()
query = cls._build_query_nodes(team.kafka_topic, label, random, name,
limit)
results = neo.query(query, returns=(str, ))
return [r[0] for r in results]
def get_label_node(cls, team_id, label, node):
team = TeamController._get({"Team": {"id": team_id}})
neo = Neo4jClient()
query = "MATCH(n:{0}_{1}{{name: '{2}'}}) RETURN n".format(
team.kafka_topic, label, node)
result = neo.query(query)
try:
data = list(result)[0][0]["data"]
except IndexError:
raise NotFoundError("Node {} does not exist".format(node))
return data
def get_impacted_nodes_count(cls, team_id, label, node, impacted_label):
if not impacted_label:
raise IntegrityError("'impactedLabel' parameter must not be empty")
team = TeamController._get({"Team": {"id": team_id}})
query = cls._build_impacted_nodes_queries(
topic=team.kafka_topic,
label=label,
node=node,
impacted_label=impacted_label,
count=True,
)
results = get_records(query)
return {"count": results.value()[0]}
def get_impacted_nodes(cls, team_id, label, node, impacted_label, skip,
limit, ts):
try:
skip = int(skip)
limit = int(limit)
ts = int(ts)
except (TypeError, ValueError):
raise IntegrityError(
"'skip', 'limit' and 'ts' parameters must be positive numbers")
if skip < 0 or limit < 0 or ts < 0:
raise IntegrityError(
"'skip', 'limit' and 'ts' parameters must be positive numbers")
if not impacted_label:
raise IntegrityError("'impactedLabel' parameter must not be empty")
team = TeamController._get({"Team": {"id": team_id}})
query = cls._build_impacted_nodes_queries(
topic=team.kafka_topic,
label=label,
node=node,
impacted_label=impacted_label,
skip=skip,
limit=limit,
count=False,
)
results = get_records(query)
impacted_nodes_data = results.data()
# Return all impacted nodes (active and inactive) with metadata indicating if they are active or not
return cls._compute_impacted_nodes_from_data(impacted_nodes_data,
ts,
with_inactive_nodes=True)
def get_node_dependencies(
cls,
team_id,
label,
node,
day=None,
filter_on_config=False,
include_inactive=False,
):
topic = TeamController._get({"Team": {"id": team_id}}).kafka_topic
query = cls._build_dependencies_query(team_id, topic, label, node,
filter_on_config)
dependencies = {
"dependencies": {},
"graph": {
"nodes": [],
"relationships": []
}
}
records = get_records(query)
# Loop on all relationships
for idx, record in enumerate(records):
# Handle the main node
if idx == 0:
node = record.get("n")
title = list(node.labels)[0][len(topic) + 1:]
if title not in dependencies["dependencies"]:
dependencies["dependencies"][title] = []
dependencies["dependencies"][title].append(dict(node.items()))
dependencies["graph"]["nodes"].append({
"id":
node.id,
"label":
dict(node.items())["name"],
"title":
title
})
# Handle the relationship
rel = record.get("r")
if not rel:
continue
# Check inactive nodes
start_node = rel.start_node
end_node = rel.end_node
start = arrow.get(day, "YYYY-MM-DD").floor("day").timestamp
end = arrow.get(day, "YYYY-MM-DD").ceil("day").timestamp
if (not is_active_node(start, end,
end_node)) or (not has_active_relationship(
start, end, rel.get("periods"))):
if not include_inactive:
continue
else:
setattr(end_node, "inactive", True)
# The label is 'acme_Mylabel', we just want 'Mylabel'
title = list(end_node.labels)[0][len(topic) + 1:]
if title not in dependencies["dependencies"]:
dependencies["dependencies"][title] = []
dependencies["dependencies"][title].append({
**dict(end_node.items()),
**{
"periods": list(rel.get("periods")),
"inactive": getattr(end_node, "inactive", False),
},
})
dependencies["graph"]["nodes"].append({
"id":
end_node.id,
"label":
dict(end_node.items())["name"],
"title":
title,
})
dependencies["graph"]["relationships"].append({
"id":
rel.id,
"from":
start_node.id,
"to":
end_node.id,
"arrows":
"to",
"periods":
list(rel.get("periods")),
})
return dependencies
def get_labels(cls, team_id):
"""
This function returns the list of labels for a team. The labels can be
found in 2 places : in the configuration or directly in Neo4j.
Having a label in Neo4j does not mean it's present in the config, and
having a label in the config does not mean it has been created in Neo4j.
"""
team = TeamController._get({"Team": {"id": team_id}})
# Get labels per team
neo = Neo4jClient()
query = """CALL db.labels() YIELD label
WITH split(label, '_')[0] AS Team, split(label, '_')[1] AS Label
RETURN Team, collect(Label) AS Labels
ORDER BY Team"""
results = neo.query(query, returns=(str, list))
# Get the label for the current team
labels_per_teams = {r[0]: r[1] for r in results}
# Pick the wanted team's labels
team_labels = labels_per_teams.get(team.kafka_topic, [])
# Construct the cypher to get the number of nodes per label
# TODO : add cache for this feature
labels = {}
if team_labels:
query = ("MATCH (n:{team}_{label}) "
'WITH "{label}" AS Label, count(n) AS Count '
"RETURN Label, Count")
query = query.format(team=team.kafka_topic, label=team_labels[0])
for label in team_labels[1:]:
union = ("\nUNION MATCH (n:{team}_{label}) "
'WITH "{label}" AS Label, count(n) AS Count '
"RETURN Label, Count")
query += union.format(team=team.kafka_topic, label=label)
results = neo.query(query, returns=(str, int))
# Merge the Neo4j and DB labels
labels = {
r[0]: {
"name": r[0],
"nodes_count": r[1],
"qos_query": None
}
for r in results
}
try:
config = ConfigController.get_current_config(team_id)
except NotFoundError:
return list(labels.values())
qos_queries = {l: d["qos"] for l, d in config["data"].items()}
for label, query in qos_queries.items():
if label not in labels:
labels[label] = {"name": label, "nodes_count": 0}
labels[label]["qos_query"] = query
return list(labels.values())
def get_node_dependencies(cls,
team_id,
label,
node,
filter_on_config=False):
team = TeamController._get({"Team": {"id": team_id}})
query = """
MATCH(n:{topic}_{label}{{name: "{name}"}})
OPTIONAL MATCH (n)-[r]->(m) {where}
RETURN n,r,m
ORDER BY m.name LIMIT 10
"""
# Filter the dependencies with the labels declared in the configuration
where = ""
if filter_on_config:
label_config = ConfigController.get_label_config(team_id, label)
regex = r"^.*(\[[A-Za-z]+(, [A-Za-z]+)*?\])$"
match = re.search(regex, label_config["qos"])
if match:
deps = match.group(1)[1:-1].split(", ")
where += "WHERE '{}_{}' IN LABELS(m)".format(
team.kafka_topic, deps[0])
for dep in deps[1:]:
where += " OR '{}_{}' IN LABELS(m)".format(
team.kafka_topic, dep)
neo = Neo4jClient()
query = query.format(topic=team.kafka_topic,
label=label,
name=node,
where=where)
sequence = neo.query(query, data_contents=DATA_GRAPH)
dependencies = {
"dependencies": {},
"graph": {
"nodes": [],
"relationships": []
}
}
if sequence.graph:
for g in sequence.graph:
# Handle the nodes
for node in g["nodes"]:
title = node["labels"][0][len(team.kafka_topic) + 1:]
if title not in dependencies["dependencies"]:
dependencies["dependencies"][title] = []
# Id is required to make data unique later
node["properties"]["id"] = node["id"]
dependencies["dependencies"][title].append(
node["properties"])
# Add the node data in the whole graph
dependencies["graph"]["nodes"].append({
"id":
node["id"],
"label":
node["properties"].get("name", "unknown"),
"title":
title,
})
# Handle the relationships
for rel in g["relationships"]:
dependencies["graph"]["relationships"].append({
"id":
rel["id"],
"from":
rel["startNode"],
"to":
rel["endNode"],
"arrows":
"to",
})
dependencies = cls.make_unique(dependencies)
return dependencies