def predict_arrival_rates(
self,
topology_id: str,
cluster: str,
environ: str,
spout_traffic: Dict[int, Dict[str, float]],
start: dt.datetime,
end: dt.datetime,
metric_bucket_length: int,
topology_ref: str = None,
**kwargs: Any) -> Tuple[pd.DataFrame, pd.DataFrame]:
""" This method will predict the arrival rate, due to the supplied spout traffic,
at each of the instances in the specified topology. These Calculations will be
based on metrics recorded during the period specified by the start and end
timestamps.
"""
if not topology_ref:
# Get the reference of the latest physical graph entry for this
# topology, or create a physical graph if there are non.
topology_ref = graph_check(
self.graph_client,
self.config,
self.tracker_url,
cluster,
environ,
topology_id,
)
# Predict Arrival Rates for all elements
instance_ars: pd.DataFrame
strmgr_ars: pd.DataFrame
instance_ars, strmgr_ars = arrival_rates.calculate(
self.graph_client, self.metrics_client, topology_id, cluster,
environ, topology_ref, start, end, metric_bucket_length,
self.tracker_url, spout_traffic, **kwargs)
# Sum the arrivals from each source component of each incoming stream
instance_ars.groupby(["task", "incoming_stream"]).sum()
in_ars: pd.DataFrame = (instance_ars.groupby([
"task", "incoming_stream"
]).sum().reset_index().rename(index=str,
columns={"incoming_stream": "stream"}))
return in_ars, strmgr_ars
def predict_current_performance(self, topology_id: str, cluster: str,
environ: str,
spout_traffic: Dict[int, Dict[str, float]],
**kwargs: Any) -> pd.DataFrame:
"""
Arguments:
topology_id (str): The topology identification string
spout_traffic (dict): The expected output of the spout instances.
These emit values should be in tuples per
second (tps) otherwise they will not match
with the service time measurements.
"""
# TODO: check spout traffic keys are integers!
start, end = _check_start_end(**kwargs)
metric_bucket_length: int = cast(int,
self.config["metric.bucket.length"])
LOG.info(
"Predicting weather back pressure will be triggered in currently running "
"topology %s ",
topology_id,
)
# Remove the start and end time kwargs so we don't supply them twice to
# the metrics client.
# TODO: We need to make this cleaner? Add start and end to topo model?
other_kwargs: Dict[str, Any] = {
key: value
for key, value in kwargs.items() if key not in ["start", "end"]
}
# Get the service time for all elements
service_times: pd.DataFrame = self.metrics_client.get_service_times(
topology_id, cluster, environ, start, end, **other_kwargs)
# Calculate the service rate for each instance
service_times["tuples_per_sec"] = 1.0 / (service_times["latency_ms"] /
1000.0)
# Drop the system streams
service_times = service_times[~service_times["stream"].str.
contains("__")]
# Calculate the median service time and rate
service_time_summary: pd.DataFrame = (service_times[[
"task", "stream", "latency_ms", "tuples_per_sec"
]].groupby(["task", "stream"]).median().reset_index())
# Get the reference of the latest physical graph entry for this
# topology, or create a physical graph if there are non.
topology_ref: str = graph_check(
self.graph_client,
self.config,
self.tracker_url,
cluster,
environ,
topology_id,
)
# Predict the arrival rate at all instances with the supplied spout
# traffic
in_ars, strmgr_ars = self.predict_arrival_rates(
topology_id,
cluster,
environ,
spout_traffic,
start,
end,
metric_bucket_length,
topology_ref,
)
combined: pd.DataFrame = service_time_summary.merge(
in_ars, on=["task", "stream"])
combined["capacity"] = (combined["arrival_rate"] /
combined["tuples_per_sec"]) * 100.0
combined["back_pressure"] = combined["capacity"] > 100.0
return combined
def get(self) -> Tuple[Dict[str, Any], int]:
# Make sure we have the args we need
errors: List[Dict[str, str]] = []
if "topology_id" not in request.args:
errors.append({
"type":
"MissingParameter",
"error": ("'topology_id' parameter should be "
"supplied"),
})
if "cluster" not in request.args:
errors.append({
"type": "MissingParameter",
"error": "'cluster' parameter should be supplied",
})
if "environ" not in request.args:
errors.append({
"type": "MissingParameter",
"error": "'environ' parameter should be supplied",
})
if "model" not in request.args:
errors.append({
"type":
"MissingParameter",
"error": ("At least one 'model' parameter should "
"be supplied. Supply 'all' to run all "
"configured models"),
})
# Return useful errors to the client if any parameters are missing
if errors:
return {"errors": errors}, 400
LOG.info(
"Traffic prediction requested for Heron topology: %s on "
"cluster: %s in environment: %s",
request.args["topology_id"],
request.args["cluster"],
request.args["environ"],
)
# Make sure we have a current graph representing the physical plan for
# the topology
try:
graph_check(
self.graph_client,
self.model_config,
self.tracker_url,
request.args["cluster"],
request.args["environ"],
request.args["topology_id"],
)
except Exception as err:
LOG.error(
"Error running graph check for topology: %s -> %s",
request.args["topology_id"],
str(err),
)
errors.append({
"topology": request.args["topology_id"],
"type": str(type(err)),
"error": str(err),
})
return {"errors": errors}, 400
output: Dict[str, Any] = {}
output["errors"] = {}
output["results"] = {}
if "all" in request.args.getlist("model"):
LOG.info("Running all configured Heron traffic performance models")
models = self.models.keys()
else:
models = request.args.getlist("model")
# Convert the request.args to a dict suitable for passing as **kwargs
model_kwargs: Dict[str, Any] = utils.convert_wimd_to_dict(request.args)
# Remove the models list from the kwargs as it is only needed by this
# method, same with topology_id, cluster and environ values
model_kwargs.pop("model")
model_kwargs.pop("topology_id")
model_kwargs.pop("cluster")
model_kwargs.pop("environ")
output = {}
for model_name in models:
LOG.info("Running traffic performance model %s", model_name)
try:
model: HeronTrafficModel = self.models[model_name]
except KeyError as key_err:
LOG.error("No such model: %s", model_name)
errors.append({
"model": model_name,
"type": str(type(key_err)),
"error": f"No such model: {str(key_err)}",
})
break
try:
results: Dict[str, Any] = model.predict_traffic(
topology_id=request.args.get("topology_id"),
cluster=request.args.get("cluster"),
environ=request.args.get("environ"),
**model_kwargs,
)
except Exception as err:
LOG.error("Error running model: %s -> %s", model.name,
str(err))
errors.append({
"model": model.name,
"type": str(type(err)),
"error": str(err)
})
output[model_name] = results
if errors:
return {"errors": errors}, 500
return output, 200
return output