def filter_records(self, start, end, records, name, dependencies):
"""
This function transforms the data returned by Neo4j into a format we
can use to search data in our TimeSeries database.
We also take this opportunity to filter inactive nodes or nodes with
inactive relationship.
"""
data = {}
for record in records:
node = record.get(name)
# Remove inactive nodes
if not is_active_node(start, end, node):
self.log.info("{node} is no longer active, skip it.".format(
node=node["name"]))
continue
# Transform the records
data[node["name"]] = []
for dependency in dependencies:
# For each dependency in this label
for n in record.get(dependency["node"]) or []:
# Do not use inactive dependencies or dependencies with inactive relation
if not is_active_node(
start, end,
n["node"]) or not has_active_relationship(
start, end, n["rel"]["periods"]):
self.log.info(
"Dependency {node} is no longer active, skip it.".
format(node=n["node"]["name"]))
continue
# This dependency can be used to compute the QOS of the main node
data[node["name"]].append({
"label": dependency["label"],
"name": n["node"]["name"]
})
# Remove nodes without any dependency
data = {k: v for k, v in data.items() if v}
return data
def get_node_dependencies(
cls,
team_id,
label,
node,
day=None,
filter_on_config=False,
include_inactive=False,
):
topic = TeamController._get({"Team": {"id": team_id}}).kafka_topic
query = cls._build_dependencies_query(team_id, topic, label, node,
filter_on_config)
dependencies = {
"dependencies": {},
"graph": {
"nodes": [],
"relationships": []
}
}
records = get_records(query)
# Loop on all relationships
for idx, record in enumerate(records):
# Handle the main node
if idx == 0:
node = record.get("n")
title = list(node.labels)[0][len(topic) + 1:]
if title not in dependencies["dependencies"]:
dependencies["dependencies"][title] = []
dependencies["dependencies"][title].append(dict(node.items()))
dependencies["graph"]["nodes"].append({
"id":
node.id,
"label":
dict(node.items())["name"],
"title":
title
})
# Handle the relationship
rel = record.get("r")
if not rel:
continue
# Check inactive nodes
start_node = rel.start_node
end_node = rel.end_node
start = arrow.get(day, "YYYY-MM-DD").floor("day").timestamp
end = arrow.get(day, "YYYY-MM-DD").ceil("day").timestamp
if (not is_active_node(start, end,
end_node)) or (not has_active_relationship(
start, end, rel.get("periods"))):
if not include_inactive:
continue
else:
setattr(end_node, "inactive", True)
# The label is 'acme_Mylabel', we just want 'Mylabel'
title = list(end_node.labels)[0][len(topic) + 1:]
if title not in dependencies["dependencies"]:
dependencies["dependencies"][title] = []
dependencies["dependencies"][title].append({
**dict(end_node.items()),
**{
"periods": list(rel.get("periods")),
"inactive": getattr(end_node, "inactive", False),
},
})
dependencies["graph"]["nodes"].append({
"id":
end_node.id,
"label":
dict(end_node.items())["name"],
"title":
title,
})
dependencies["graph"]["relationships"].append({
"id":
rel.id,
"from":
start_node.id,
"to":
end_node.id,
"arrows":
"to",
"periods":
list(rel.get("periods")),
})
return dependencies
def execute(self, context):
from depc.controllers import NotFoundError
from depc.controllers.rules import RuleController
from depc.extensions import redis_scheduler as redis
from depc.utils import get_start_end_ts
ds = context["ds"]
start, end = get_start_end_ts(ds)
with self.app.app_context():
# Get the nodes for this team and this label
query = ("MATCH(n:{label}) RETURN n AS Node "
"ORDER BY Node.name "
"SKIP {skip} LIMIT {limit}")
query = query.format(label=self.full_label,
skip=self.skip,
limit=int(self.length))
records = get_records(query)
nodes = [dict(record.get("Node").items()) for record in records]
# Remove old nodes
nodes = [n for n in nodes if is_active_node(start, end, n)]
# Get the rule associated to the label for this team
try:
rule = RuleController.get(filters={
"Rule": {
"name": self.rule_name,
"team_id": self.team_id
}
})
except NotFoundError:
self.log.warning(
"[{0}] The label {1} has no associated rule in DEPC".
format(self.team_name, self.label))
return False
has_qos = False
auto_fill = check_enable_auto_fill(rule["id"], self.team_id)
for node in nodes:
result = RuleController.execute(
rule_id=rule["id"],
auto_fill=auto_fill,
name=node["name"],
start=start,
end=end,
)
if result["qos"]["qos"] != "unknown":
has_qos = True
self.log.info("[{0}/{1}] The QOS of {2} is {3}%".format(
self.team_name,
self.label,
node["name"],
result["qos"]["qos"],
))
# Saving to Beamium
self.write_metric(
metric="depc.qos.node",
ts=start,
value=result["qos"]["qos"],
tags={
"label": self.label,
"name": node["name"],
"team": self.team_id,
},
)
# Used for average computing
key = "{ds}.{team}.{label}".format(ds=ds,
team=self.team_name,
label=self.label)
if not self.excluded_from_label_average(
self.team_name, self.label, node["name"]):
redis.zadd("{}.sorted".format(key), node["name"],
result["qos"]["qos"])
# Save information to reuse it later (`bools_dps` is used in
# OperationOperator and `qos` is used in AggregationOperator)
redis.set(
"{}.{}.node".format(key, node["name"]),
json.dumps({
"bools_dps": result["qos"]["bools_dps"],
"qos": result["qos"]["qos"],
}),
)
else:
self.log.warning("[{0}/{1}] No QOS for {2}".format(
self.team_name, self.label, node["name"]))
# Add it in redis to compute some stats in AfterSubdagOperator
redis.sadd(
"{ds}.{team}.{label}.noqos".format(ds=ds,
team=self.team_name,
label=self.label),
node["name"],
)
if not has_qos:
self.log.warning("[{0}/{1}] No QOS found for any items".format(
self.team_name, self.label))