class Event: def __init__(self, want_max=True, want_mean=True, want_stdev=True, want_min=True): self.stat = Statistics() self.want_max = want_max self.want_mean = want_mean self.want_stdev = want_stdev self.want_min = want_min def to_kvhf(self): mins = [self.stat.minimum()] if self.want_min else [] maxs = [self.stat.maximum()] if self.want_max else [] #This is necessary cause bug of runstat https://github.com/grantjenks/python-runstats/issues/27 if len(self.stat) > 1: stdevs = [self.stat.stddev()] if self.want_stdev else [] else: stdevs = [0] means = [self.stat.mean()] if self.want_mean else [] return Serie_stats(means=means, mins=mins, maxs=maxs, stdevs=stdevs) def add_occurence(self, time): self.stat.push(time) def get_occurence_num(self): return len(self.stat)
def test_statistics(): alpha = [random.random() for val in range(count)] alpha_stats = Statistics() for val in alpha: alpha_stats.push(val) assert len(alpha_stats) == count assert error(mean(alpha), alpha_stats.mean()) < error_limit assert error(variance(alpha), alpha_stats.variance()) < error_limit assert error(stddev(alpha), alpha_stats.stddev()) < error_limit assert error(skewness(alpha), alpha_stats.skewness()) < error_limit assert error(kurtosis(alpha), alpha_stats.kurtosis()) < error_limit assert alpha_stats.minimum() == min(alpha) assert alpha_stats.maximum() == max(alpha) alpha_stats.clear() assert len(alpha_stats) == 0 alpha_stats = Statistics(alpha) beta = [random.random() for val in range(count)] beta_stats = Statistics() for val in beta: beta_stats.push(val) gamma_stats = alpha_stats + beta_stats assert len(beta_stats) != len(gamma_stats) assert error(mean(alpha + beta), gamma_stats.mean()) < error_limit assert error(variance(alpha + beta), gamma_stats.variance()) < error_limit assert error(stddev(alpha + beta), gamma_stats.stddev()) < error_limit assert error(skewness(alpha + beta), gamma_stats.skewness()) < error_limit assert error(kurtosis(alpha + beta), gamma_stats.kurtosis()) < error_limit assert gamma_stats.minimum() == min(alpha + beta) assert gamma_stats.maximum() == max(alpha + beta) delta_stats = beta_stats.copy() delta_stats += alpha_stats assert len(beta_stats) != len(delta_stats) assert error(mean(alpha + beta), delta_stats.mean()) < error_limit assert error(variance(alpha + beta), delta_stats.variance()) < error_limit assert error(stddev(alpha + beta), delta_stats.stddev()) < error_limit assert error(skewness(alpha + beta), delta_stats.skewness()) < error_limit assert error(kurtosis(alpha + beta), delta_stats.kurtosis()) < error_limit assert delta_stats.minimum() == min(alpha + beta) assert delta_stats.maximum() == max(alpha + beta)
class Client: # This comes from Contiki-NG's circular buffer # Currently there is no way to increase this value max_serial_len = 127 task_stats_prefix = f"app{serial_sep}stats{serial_sep}" def __init__(self, name, task_runner, max_workers=2): self.name = name self.reader = None self.writer = None self.message_prefix = f"{application_edge_marker}{self.name}{serial_sep}" self.stats = Statistics() self.executor = ProcessPoolExecutor(max_workers=max_workers) self._task_runner = task_runner self.ack_cond = asyncio.Condition() self.response_lock = asyncio.Lock() self.was_cancelled = False async def start(self): self.reader, self.writer = await asyncio.open_connection( 'localhost', edge_server_port) # Need to inform bridge of what application we represent await self.write(f"{self.name}\n") # Wait until we get the ready message line = await self.reader.readline() # Check if the endpoint closed on us if not line: logger.info("Connection closed while waiting for ready") return line = line.decode("utf-8").rstrip() if line != "ready": raise RuntimeError(f"Unexpected start message {line}") # Once started, we need to inform the edge of this application's availability await self._inform_application_started() async def run(self): while not self.reader.at_eof(): line = await self.reader.readline() # Check if the endpoint closed on us if not line: logger.info("Connection closed") break line = line.decode("utf-8").rstrip() # Process ack if line.endswith(f"{serial_sep}ack"): async with self.ack_cond: self.ack_cond.notify() continue # Process cancel if line.endswith(f"{serial_sep}cancel"): self.was_cancelled = True continue # Create task here to allow multiple jobs from clients to be # processed simultaneously (if they wish) asyncio.create_task(self.receive(line)) async def stop(self): self.executor.shutdown() # When stopping, we need to inform the edge that this application is no longer available await self._inform_application_stopped() self.writer.close() await self.writer.wait_closed() self.reader = None self.writer = None async def receive(self, message: str): try: dt, src, payload_len, payload = message.split(serial_sep, 3) dt = datetime.fromisoformat(dt) src = ipaddress.IPv6Address(src) payload_len = int(payload_len) payload = bytes.fromhex(payload) if len(payload) != payload_len: logger.error( f"Incorrect payload length, expected {payload_len}, actual {len(payload)}" ) return payload = cbor2.loads(payload) logger.debug( f"Received task at {dt} from {src} <payload={payload}>") except Exception as ex: logger.error(f"Failed to parse message '{message}' with {ex}") return try: loop = asyncio.get_running_loop() task_result = await loop.run_in_executor(self.executor, self._task_runner, (src, dt, payload)) except Exception as ex: logger.error( f"Failed to execute task '{(src, dt, payload)}' with {ex}") logger.error(traceback.format_exc()) # Send the internal error back to this device # Only 1 response can be sent at a given time async with self.response_lock: await self._send_result(src, self.internal_error) return (dest, message_response, duration) = task_result # Update the average time taken to perform jobs # TODO: should this be EWMA? self.stats.push(duration) # Only 1 response can be sent at a given time async with self.response_lock: await self._send_result(dest, message_response) async def _send_result(self, dest, message_response): raise NotImplementedError() async def _receive_ack(self): async with self.ack_cond: await self.ack_cond.wait() def _check_and_reset_cancelled(self) -> bool: result = self.was_cancelled self.was_cancelled = False return result async def write(self, message: str): logger.debug(f"Writing {message!r} of length {len(message)}") encoded_message = message.encode("utf-8") if len(encoded_message) > self.max_serial_len: logger.warn( f"Encoded message is longer ({len(encoded_message)}) than the maximum allowed length ({self.max_serial_len}) it will be truncated" ) self.writer.write(encoded_message) await self.writer.drain() async def _write_to_application(self, message: str, application_name: Optional[str] = None): # By default send this message to the application this process represents if not application_name: application_name = self.name await self.write( f"{application_edge_marker}{application_name}{serial_sep}{message}\n" ) async def _inform_application_started( self, application_name: Optional[str] = None): await self._write_to_application("start", application_name=application_name) async def _inform_application_stopped( self, application_name: Optional[str] = None): await self._write_to_application("stop", application_name=application_name) async def _write_task_stats(self): await self._write_to_application( f"{self.task_stats_prefix}{self._stats_string()}") await self._receive_ack() def _stats_string(self) -> str: try: variance = int(math.ceil(self.stats.variance())) except ZeroDivisionError: variance = 0 mean = int(math.ceil(self.stats.mean())) maximum = int(math.ceil(self.stats.maximum())) minimum = int(math.ceil(self.stats.minimum())) data = (mean, maximum, minimum, variance) return base64.b64encode(cbor2.dumps(data)).decode("utf-8")