def predict_arrival_rates( self, topology_id: str, cluster: str, environ: str, spout_traffic: Dict[int, Dict[str, float]], start: dt.datetime, end: dt.datetime, metric_bucket_length: int, topology_ref: str = None, **kwargs: Any) -> Tuple[pd.DataFrame, pd.DataFrame]: """ This method will predict the arrival rate, due to the supplied spout traffic, at each of the instances in the specified topology. These Calculations will be based on metrics recorded during the period specified by the start and end timestamps. """ if not topology_ref: # Get the reference of the latest physical graph entry for this # topology, or create a physical graph if there are non. topology_ref = graph_check( self.graph_client, self.config, self.tracker_url, cluster, environ, topology_id, ) # Predict Arrival Rates for all elements instance_ars: pd.DataFrame strmgr_ars: pd.DataFrame instance_ars, strmgr_ars = arrival_rates.calculate( self.graph_client, self.metrics_client, topology_id, cluster, environ, topology_ref, start, end, metric_bucket_length, self.tracker_url, spout_traffic, **kwargs) # Sum the arrivals from each source component of each incoming stream instance_ars.groupby(["task", "incoming_stream"]).sum() in_ars: pd.DataFrame = (instance_ars.groupby([ "task", "incoming_stream" ]).sum().reset_index().rename(index=str, columns={"incoming_stream": "stream"})) return in_ars, strmgr_ars
def predict_current_performance(self, topology_id: str, cluster: str, environ: str, spout_traffic: Dict[int, Dict[str, float]], **kwargs: Any) -> pd.DataFrame: """ Arguments: topology_id (str): The topology identification string spout_traffic (dict): The expected output of the spout instances. These emit values should be in tuples per second (tps) otherwise they will not match with the service time measurements. """ # TODO: check spout traffic keys are integers! start, end = _check_start_end(**kwargs) metric_bucket_length: int = cast(int, self.config["metric.bucket.length"]) LOG.info( "Predicting weather back pressure will be triggered in currently running " "topology %s ", topology_id, ) # Remove the start and end time kwargs so we don't supply them twice to # the metrics client. # TODO: We need to make this cleaner? Add start and end to topo model? other_kwargs: Dict[str, Any] = { key: value for key, value in kwargs.items() if key not in ["start", "end"] } # Get the service time for all elements service_times: pd.DataFrame = self.metrics_client.get_service_times( topology_id, cluster, environ, start, end, **other_kwargs) # Calculate the service rate for each instance service_times["tuples_per_sec"] = 1.0 / (service_times["latency_ms"] / 1000.0) # Drop the system streams service_times = service_times[~service_times["stream"].str. contains("__")] # Calculate the median service time and rate service_time_summary: pd.DataFrame = (service_times[[ "task", "stream", "latency_ms", "tuples_per_sec" ]].groupby(["task", "stream"]).median().reset_index()) # Get the reference of the latest physical graph entry for this # topology, or create a physical graph if there are non. topology_ref: str = graph_check( self.graph_client, self.config, self.tracker_url, cluster, environ, topology_id, ) # Predict the arrival rate at all instances with the supplied spout # traffic in_ars, strmgr_ars = self.predict_arrival_rates( topology_id, cluster, environ, spout_traffic, start, end, metric_bucket_length, topology_ref, ) combined: pd.DataFrame = service_time_summary.merge( in_ars, on=["task", "stream"]) combined["capacity"] = (combined["arrival_rate"] / combined["tuples_per_sec"]) * 100.0 combined["back_pressure"] = combined["capacity"] > 100.0 return combined
def get(self) -> Tuple[Dict[str, Any], int]: # Make sure we have the args we need errors: List[Dict[str, str]] = [] if "topology_id" not in request.args: errors.append({ "type": "MissingParameter", "error": ("'topology_id' parameter should be " "supplied"), }) if "cluster" not in request.args: errors.append({ "type": "MissingParameter", "error": "'cluster' parameter should be supplied", }) if "environ" not in request.args: errors.append({ "type": "MissingParameter", "error": "'environ' parameter should be supplied", }) if "model" not in request.args: errors.append({ "type": "MissingParameter", "error": ("At least one 'model' parameter should " "be supplied. Supply 'all' to run all " "configured models"), }) # Return useful errors to the client if any parameters are missing if errors: return {"errors": errors}, 400 LOG.info( "Traffic prediction requested for Heron topology: %s on " "cluster: %s in environment: %s", request.args["topology_id"], request.args["cluster"], request.args["environ"], ) # Make sure we have a current graph representing the physical plan for # the topology try: graph_check( self.graph_client, self.model_config, self.tracker_url, request.args["cluster"], request.args["environ"], request.args["topology_id"], ) except Exception as err: LOG.error( "Error running graph check for topology: %s -> %s", request.args["topology_id"], str(err), ) errors.append({ "topology": request.args["topology_id"], "type": str(type(err)), "error": str(err), }) return {"errors": errors}, 400 output: Dict[str, Any] = {} output["errors"] = {} output["results"] = {} if "all" in request.args.getlist("model"): LOG.info("Running all configured Heron traffic performance models") models = self.models.keys() else: models = request.args.getlist("model") # Convert the request.args to a dict suitable for passing as **kwargs model_kwargs: Dict[str, Any] = utils.convert_wimd_to_dict(request.args) # Remove the models list from the kwargs as it is only needed by this # method, same with topology_id, cluster and environ values model_kwargs.pop("model") model_kwargs.pop("topology_id") model_kwargs.pop("cluster") model_kwargs.pop("environ") output = {} for model_name in models: LOG.info("Running traffic performance model %s", model_name) try: model: HeronTrafficModel = self.models[model_name] except KeyError as key_err: LOG.error("No such model: %s", model_name) errors.append({ "model": model_name, "type": str(type(key_err)), "error": f"No such model: {str(key_err)}", }) break try: results: Dict[str, Any] = model.predict_traffic( topology_id=request.args.get("topology_id"), cluster=request.args.get("cluster"), environ=request.args.get("environ"), **model_kwargs, ) except Exception as err: LOG.error("Error running model: %s -> %s", model.name, str(err)) errors.append({ "model": model.name, "type": str(type(err)), "error": str(err) }) output[model_name] = results if errors: return {"errors": errors}, 500 return output, 200 return output