def get_team_qos(cls, team_id, label, start, end):
cls.check_valid_period(start, end)
client = Warp10Client()
# Default values
params = {"team": team_id}
script = QOS_PER_TEAM
if label:
script = QOS_FILTERED_BY_LABEL
params.update({"name": label})
client.generate_script(start=start, end=end, script=script, extra_params=params)
resp = client.execute()
# Return list of datapoints
if label:
try:
return _transform_warp10_values(resp[0][0]["v"])
except IndexError:
return {}
# Order by Label
data = {}
for metrics in resp[0]:
label = metrics["l"]["name"]
if label not in data:
data[label] = {}
data[label] = _transform_warp10_values(metrics["v"])
return data
def get_label_worst_items(cls, team, label, start, end, count):
cls.check_valid_period(start, end)
client = Warp10Client()
client.generate_script(
start=start,
end=end,
script=QOS_WORST_ITEM_PER_LABEL,
extra_params={
"team": team,
"label": label,
"count": str(count)
},
)
resp = client.execute()
# No result
if not resp[0]:
return {}
data = []
for metric in resp[0]:
data.append({
"name": metric["l"]["name"],
"qos": metric["v"][0][1]
})
return data
def execute(self, parameters, name, start, end):
client = Warp10Client(url=self.configuration["url"],
rotoken=self.configuration["token"])
# Generate the WarpScript and
# change the placeholders.
client.generate_script(start=start.timestamp,
end=end.timestamp,
script=parameters["script"])
try:
response = client.execute()
except Warp10Exception as e:
try:
message = html.unescape(
re.search("<pre>(.*)<\/pre>", str(e)).groups()[0].strip())
except Exception:
message = str(e)
raise BadConfigurationException(
"Warp10 Internal Error : {0}".format(message))
# Transform the Warp10 values
timeseries = []
for ts in response[0]:
timeseries.append({
"dps": _transform_warp10_values(ts["v"]),
"metric": ts["c"],
"tags": ts["l"],
})
if not timeseries:
msg = "No data for {} (from {} to {})".format(name, start, end)
logger.warning(msg)
raise UnknownStateException(msg)
# Parse the threshold
try:
top_threshold = float(parameters["top_threshold"])
bottom_threshold = float(parameters["bottom_threshold"])
except ValueError:
msg = "Thresholds are not valid (must be floats): {} and {}".format(
parameters["top_threshold"], parameters["bottom_threshold"])
raise BadConfigurationException(msg)
# Compute the QoS
try:
result = compute_interval(
timeseries,
start.timestamp,
end.timestamp,
bottom_threshold,
top_threshold,
)
except DataFetchException as e:
raise UnknownStateException(e)
result.update({"timeseries": timeseries})
return result
def execute(self, context):
from depc.controllers import NotFoundError
from depc.controllers.worst import WorstController
from depc.models.worst import Periods
from depc.utils import get_start_end_ts
from depc.utils.warp10 import Warp10Client
# get the right start and end with the date of the DAG
ds = context["ds"]
start, end = get_start_end_ts(ds)
with self.app.app_context():
client = Warp10Client(use_cache=True)
client.generate_script(
WORST_ITEMS_PER_LABEL,
start,
end,
max_gts=MAX_DATAPOINTS,
extra_params={
"team": self.team_id,
"label": self.label,
"count": str(MAX_WORST_ITEMS),
},
)
start = time.time()
results = client.execute()
self.log.info("Warp10 script took {}s".format(
round(time.time() - start, 3)))
# produce the data for the relational database
worst_items = {}
for d in results[0]:
qos = d["v"][0][1]
if qos < 100:
worst_items[d["l"]["name"]] = qos
nb_worst_items = len(worst_items)
if nb_worst_items:
self.log.info("Worst: {} item(s)".format(nb_worst_items))
metadata = {
"team_id": self.team_id,
"label": self.label,
"period": Periods.daily,
"date": ds,
}
try:
WorstController.update(data={"data": worst_items},
filters={"Worst": metadata})
except NotFoundError:
payload = {"data": worst_items}
payload.update(metadata)
WorstController.create(payload)
else:
self.log.info("No QOS under 100%")
def get_team_item_qos(cls, team, label, name, start, end):
cls.check_valid_period(start, end)
client = Warp10Client()
client.generate_script(
start=start,
end=end,
script=QOS_ITEM_PER_TEAM_LABEL,
extra_params={"team": team, "label": label, "name": name},
)
resp = client.execute()
# No result
if not resp[0]:
return {}
values = _transform_warp10_values(resp[0][0]["v"])
return values
def get_team_statistics(cls,
team_id,
start,
end,
label=None,
type=None,
sort="label"):
cls.check_valid_period(start, end)
client = Warp10Client()
filter_str = " "
if label:
filter_str += "'label' '{0}' ".format(label)
if type:
filter_str += "'type' '{0}' ".format(type)
client.generate_script(
start=start,
end=end,
script=STATISTICS_SCRIPT,
extra_params={
"team": team_id,
"filter": filter_str
},
)
resp = client.execute()
# No result
if not resp[0]:
return {}
# Sort the results
sort_choices = ["label", "type"]
main_sort = sort_choices.pop(
sort_choices.index(sort if sort in sort_choices else "label"))
second_sort = sort_choices.pop()
result = {}
# We want to have some statistics for all labels
# (ex: total number of nodes for the team)
if sort == "label":
result["*"] = {}
for statistic in resp[0]:
main = statistic["l"][main_sort]
second = statistic["l"][second_sort]
if main not in result:
result[main] = {}
if second not in result[main]:
result[main][second] = {}
# Total of stats for all labels
if sort == "label":
if second not in result["*"]:
result["*"][second] = {}
for v in statistic["v"]:
ts = int(v[0] / 1000000) # ms to s
result[main][second][ts] = v[1]
# Increment the total
if sort == "label":
if ts not in result["*"][second]:
result["*"][second][ts] = 0
result["*"][second][ts] += v[1]
return result