def _check_config_from_context(cls, config_context, team_id):
# To avoid multiple identical calls to the database
already_checked_rules = set()
for label_name, meta in config_context["all_labels"].items():
# This is not possible to schedule an operation or a rule after an aggregation
# The only task authorized after an aggregation is also another aggregation
try:
dstream_label_name, dstream_compute_type = config_context[
"labels_with_downstream"
][label_name]
if (
config_context["all_labels"][label_name]["compute_type"]
== "aggregation"
and dstream_compute_type != "aggregation"
):
raise IntegrityError(
'Label "{dstream_label_name}" could not be executed after label "{label_name}"'.format(
dstream_label_name=dstream_label_name, label_name=label_name
)
)
except KeyError:
# It happens when the current label has no downstream compute task
# (not present into config_context['labels_with_downstream'])
pass
if (
meta["compute_type"] == "rule"
and config_context["all_labels"][label_name]["rule_name"]
not in already_checked_rules
):
# Search if the rule exists
rule_name = config_context["all_labels"][label_name]["rule_name"]
try:
RuleController.get(
filters={"Rule": {"name": rule_name, "team_id": team_id}}
)
except NotFoundError:
raise NotFoundError(
'Rule "{rule}" defined in label "{label}" does not exist'.format(
rule=rule_name, label=label_name
)
)
already_checked_rules.add(rule_name)
# This is an operation or an aggregation
elif (
meta["compute_type"] == "operation"
or meta["compute_type"] == "aggregation"
):
for dep in meta["dependencies"]:
if dep not in config_context["all_labels"].keys():
msg = (
'Dependency "{dep}" declared in label "{label}" has not '
"been declared in the configuration"
)
raise NotFoundError(msg.format(dep=dep, label=label_name))
def execute_rule(team_id, rule_id):
"""
.. :quickref: POST; Lorem ipsum."""
if not TeamPermission.is_user(team_id):
abort(403)
payload = get_payload()
# Does the team owns the rule
RuleController.get({"Rule": {"id": rule_id, "team_id": team_id}})
result = RuleController.execute(rule_id, **payload)
return jsonify({"result": result}), 200
def list_rule_checks(team_id, rule_id):
"""
.. :quickref: GET; Lorem ipsum."""
rule = RuleController.get(
filters={"Rule": {
"id": rule_id,
"team_id": team_id
}})
return jsonify({"checks": [format_check(r) for r in rule["checks"]]}), 200
def get_rule(team_id, rule_id):
"""Return a specific rule.
.. :quickref: GET; Return a specific rule.
**Example request**:
.. sourcecode:: http
GET /teams/66859c4a-3e0a-4968-a5a4-4c3b8662acb7/rules/ff130e9b-d226-4465-9612-a93e12799091 HTTP/1.1
Host: example.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
{
"checks": [],
"createdAt": "2018-05-17T12:01:09Z",
"description": "Compute the QOS of our servers",
"id": "ff130e9b-d226-4465-9612-a93e12799091",
"name": "Servers",
"updatedAt": "2018-11-09T15:33:06Z"
}
:resheader Content-Type: application/json
:status 200: the rule
"""
if not TeamPermission.is_user(team_id):
abort(403)
rule = RuleController.get(
filters={"Rule": {
"id": rule_id,
"team_id": team_id
}})
return jsonify(format_rule(rule)), 200
def execute(self, context):
from depc.controllers import NotFoundError
from depc.controllers.rules import RuleController
from depc.extensions import redis_scheduler as redis
from depc.utils import get_start_end_ts
ds = context["ds"]
start, end = get_start_end_ts(ds)
with self.app.app_context():
# Get the nodes for this team and this label
query = ("MATCH(n:{label}) RETURN n AS Node "
"ORDER BY Node.name "
"SKIP {skip} LIMIT {limit}")
query = query.format(label=self.full_label,
skip=self.skip,
limit=int(self.length))
records = get_records(query)
nodes = [dict(record.get("Node").items()) for record in records]
# Remove old nodes
nodes = [n for n in nodes if is_active_node(start, end, n)]
# Get the rule associated to the label for this team
try:
rule = RuleController.get(filters={
"Rule": {
"name": self.rule_name,
"team_id": self.team_id
}
})
except NotFoundError:
self.log.warning(
"[{0}] The label {1} has no associated rule in DEPC".
format(self.team_name, self.label))
return False
has_qos = False
auto_fill = check_enable_auto_fill(rule["id"], self.team_id)
for node in nodes:
result = RuleController.execute(
rule_id=rule["id"],
auto_fill=auto_fill,
name=node["name"],
start=start,
end=end,
)
if result["qos"]["qos"] != "unknown":
has_qos = True
self.log.info("[{0}/{1}] The QOS of {2} is {3}%".format(
self.team_name,
self.label,
node["name"],
result["qos"]["qos"],
))
# Saving to Beamium
self.write_metric(
metric="depc.qos.node",
ts=start,
value=result["qos"]["qos"],
tags={
"label": self.label,
"name": node["name"],
"team": self.team_id,
},
)
# Used for average computing
key = "{ds}.{team}.{label}".format(ds=ds,
team=self.team_name,
label=self.label)
if not self.excluded_from_label_average(
self.team_name, self.label, node["name"]):
redis.zadd("{}.sorted".format(key), node["name"],
result["qos"]["qos"])
# Save information to reuse it later (`bools_dps` is used in
# OperationOperator and `qos` is used in AggregationOperator)
redis.set(
"{}.{}.node".format(key, node["name"]),
json.dumps({
"bools_dps": result["qos"]["bools_dps"],
"qos": result["qos"]["qos"],
}),
)
else:
self.log.warning("[{0}/{1}] No QOS for {2}".format(
self.team_name, self.label, node["name"]))
# Add it in redis to compute some stats in AfterSubdagOperator
redis.sadd(
"{ds}.{team}.{label}.noqos".format(ds=ds,
team=self.team_name,
label=self.label),
node["name"],
)
if not has_qos:
self.log.warning("[{0}/{1}] No QOS found for any items".format(
self.team_name, self.label))