def __init__(self, jid: str, password: str, predictor: PlannerPredictor, executors_jid: Iterable[str], cache_max_size: int = 10, verify_security: bool = False): """Agent to predict Concept Drifts from KB using a customized Predictor for that. This agent authenticates on XMPP server. :param jid: Id for XMPP authentication. Ex: user@localhost :type jid: str :param password: Password for XMPP authentication. :type password: str :param predictor: Predictor for Concept Drift detection. :type predictor: PlannerPredictor :param executors_jid: List of executors that this planner will send detected Concept Drifts. :type executors_jid: Iterable[str] :param cache_max_size: Cache list if executor is unavailable, defaults to 10 :type cache_max_size: int, optional :param verify_security: [description], defaults to False :type verify_security: bool, optional """ self._predictor = predictor self._executors = executors_jid self._cache = deque([], cache_max_size) self._sent_data = {} self._logger = getLogger("planner") super().__init__(jid, password, verify_security)
def __init__(self, jid: str, password: str, predictor: AnalyserPredictor, database_connection: Connection, monitors_jid: Iterable[str], verify_security: bool = False): """Agent to predict Concept Drifts using a customized Predictor for that. This agent authenticates on XMPP server. :param jid: Id for XMPP authentication. Ex: user@localhost :type jid: str :param password: Password for XMPP authentication. :type password: str :param predictor: Predictor for Concept Drift detection. :type predictor: AnalyserPredictor :param database_connection: Database connection using SQLAlchemy. :type database_connection: Connection :param monitors_jid: List of monitors that this analyser will predict Concept Drifts. :type monitors_jid: Iterable[str] :param verify_security: Security validation with XMPP server, defaults to False. :type verify_security: bool, optional """ self._monitors = monitors_jid self._connection = database_connection self._predictor = predictor self._logger = getLogger("analyser") super().__init__(jid, password, verify_security)
def __init__(self, db_engine: Engine, bulk_size: int, bulk_time: Union[int, float], circuit_breaker: CircuitBreaker = CircuitBreaker()): """Connects with SQLAlchemy Engine to store and query data for concept drift datection. :param db_engine: SQLAlchemy Engine to use as backend :type db_engine: Engine :param bulk_size: Quantity of data that connection will wait to make bulk insert :type bulk_size: int :param bulk_time: Time in seconds between last insert and now. If bulk_size is not reached in bulk_time interval, then an insert was done :type bulk_time: Union[int, float] :param circuit_breaker: Circuit Breaker configuration to connect with Database, defaults to CircuitBreaker() :type circuit_breaker: CircuitBreaker, optional """ self._jid = None self._conn = db_engine self._bulk_size = bulk_size self._bulk_time = bulk_time self._last_insert_time = datetime.utcnow() self._bulk_df = pd.DataFrame() self.get_between = circuit_breaker(self.get_between) self._insert = circuit_breaker(self._insert) self._logger = getLogger("connection")
class TrainPredictor(PeriodicBehaviour): _logger = getLogger("train_predictor") async def run(self): """[summary] """ await self.agent.predictor.fit() self._logger.debug("Analyser model fitted")
class ReceiveNewData(CyclicBehaviour): _logger = getLogger("receive_new_data") async def run(self): """[summary] """ msg = await self.receive() if msg: self.agent.add_behaviour(StoreNewData(), Template(body=msg.body)) self._logger.debug(f"Message received {msg.body}")
class FastNotifyContacts(OneShotBehaviour): _logger = getLogger("fast_notify_contacts") async def run(self): """[summary] """ for contact in self.agent.available_contacts.copy(): msg = Message(to=contact, body=self.template.body) await self.send(msg) self._logger.debug(f"Sent {self.template.body} to all contacts")
class SendNewData(OneShotBehaviour): _logger = getLogger("send_new_data") async def run(self): """[summary] """ body = orjson.loads(self.template.body) for msg in body: if self.agent.sink.is_available(): await self.agent.sink.drain(msg) self._logger.debug(f"Message sent to Sink {msg}")
class ReceiveNewData(CyclicBehaviour): _logger = getLogger("receive_new_data") async def run(self): """[summary] """ if self.agent.sink.is_available(): self.presence.set_available() else: self.presence.set_unavailable() msg = await self.receive() if msg: self.agent.add_behaviour(SendNewData(), Template(body=msg.body)) self._logger.debug(f"Message received {msg.body}")
class WaitMonitorSubscriptions(WaitSubscriptions): _logger = getLogger("wait_monitor_subscriptions") def on_subscribe(self, jid: str): """[summary] :param jid: [description] :type jid: [str] """ self.presence.approve(jid) self.presence.subscribe(jid) self._logger.debug(f"Approved and subscribing to {jid}") async def run(self): """[summary] """ self.presence.on_subscribe = self.on_subscribe await super().run()
def __init__(self, jid: str, password: str, identifier: Optional[str] = None, verify_security: bool = False): """An Agent to collect data from sources and send to Analyser. This agent authenticates on XMPP server. :param jid: Id for XMPP authentication. Ex: user@localhost :type jid: str :param password: Password for XMPP authentication. :type password: str :param identifier: Data identification or agent jid, defaults to None :type identifier: Optional[str], optional :param verify_security: Security validation with XMPP server, defaults to False. :type verify_security: bool, optional """ super().__init__(jid, password, verify_security) self._identifier = identifier if identifier else self.name self._logger = getLogger("monitor")
def __init__(self, jid: str, password: str, sink: Sink, verify_security: bool = False): """Agent to send predicted drifts to Sink. This agent authenticates on XMPP server. :param jid: Id for XMPP authentication. Ex: user@localhost :type jid: str :param password: Password for XMPP authentication. :type password: str :param sink: Where predicted Concept Drifts will be dispatched. :type sink: Sink :param verify_security: Security validation with XMPP server, defaults to False. :type verify_security: bool, optional """ self._sink = sink self._logger = getLogger("executor") super().__init__(jid, password, verify_security)
class WaitSubscriptions(OneShotBehaviour): _logger = getLogger("wait_subscriptions") def on_available(self, jid: str, stanza: Presence): """[summary] :param jid: [description] :type jid: [str] :param stanza: [description] :type stanza: [Presence] """ if not jid.startswith(self.agent.jid.localpart): self.agent.available_contacts[jid] = stanza self._logger.debug(f"Contact added {jid}") def on_unavailable(self, jid: str, stanza: Presence): """[summary] :param jid: [description] :type jid: [str] :param stanza: [description] :type stanza: [Presence] """ try: del self.agent.available_contacts[jid] self._logger.debug(f"Contact removed {jid}") except KeyError: pass async def run(self): """[summary] """ self.presence.on_available = self.on_available self.presence.on_unavailable = self.on_unavailable self.presence.set_available()
class NotifyContacts(OneShotBehaviour): _logger = getLogger("notify_contacts") async def run(self): """[summary] """ for contact in self.agent.available_contacts.copy(): contact_data = self.agent.sent_data[contact] if ((len(contact_data) > 0) and (contact_data[-1] == id(self.agent.cache[-1]))): continue to_send = [] to_send_id = [] for item in self.agent.cache: if id(item) in contact_data: continue to_send.append(item) to_send_id.append(id(item)) msg = Message(to=contact, body=str(orjson.dumps(to_send), "utf-8")) await self.send(msg) self.agent.sent_data[contact].extend(to_send_id) self._logger.debug(f"Sent to contact {contact} data {to_send}")
class Predict(PeriodicBehaviour): _logger = getLogger("predict") async def run(self): """[summary] """ predictor = self.agent.predictor results = await predictor.predict() has_new_data = False for result in results: if result.should_send: self.agent.cache.append({ "timestamp": datetime.utcnow().timestamp(), "identifier": result.identifier, "predicted": result.predicted }) has_new_data = True if has_new_data: self.agent.add_behaviour(NotifyContacts()) self._logger.debug("Notified new predictions")
def test_should_return_child_log_with_new_level(self): logger = getLogger("test", "DEBUG") self.assertEqual(logger.name, "driftage.test") self.assertEqual(logger.level, 10)
def test_should_return_child_log(self): logger = getLogger("test") self.assertEqual(logger.name, "driftage.test") self.assertEqual(logger.level, 20)
class StoreNewData(OneShotBehaviour): _logger = getLogger("store_new_data") async def run(self): """[summary] """ msg = orjson.loads(self.template.body) data = await self._parse(msg) predicted = await self._predict(data) await self._store(data, predicted) self._logger.debug(f"Data stored on database {data}") async def _parse(self, msg: dict) -> pd.DataFrame: """[summary] :param msg: [description] :type msg: dict :return: [description] :rtype: PredictionData """ data = msg["data"] metadata = msg["metadata"] return PredictionData( data=data, timestamp=datetime.fromtimestamp(metadata["timestamp"]), identifier=metadata["identifier"] ) async def _predict(self, data: PredictionData) -> bool: """[summary] :param data: [description] :type data: PredictionData :return: [description] :rtype: bool """ return await self.agent.predictor.predict(data) async def _store(self, data: PredictionData, prediction: bool): """[summary] :param data: [description] :type data: PredictionData :param prediction: [description] :type prediction: bool """ df = pd.DataFrame( { table.c.driftage_jid.name: [self.agent.name], table.c.driftage_data.name: [ str(orjson.dumps(data.data), "utf-8")], table.c.driftage_datetime_monitored.name: [data.timestamp], table.c.driftage_datetime_analysed.name: [datetime.utcnow()], table.c.driftage_identifier.name: [data.identifier], table.c.driftage_predicted.name: [prediction] } ) await self.agent.connection.lazy_insert(df)