def __init__(self,
jid: str,
password: str,
predictor: PlannerPredictor,
executors_jid: Iterable[str],
cache_max_size: int = 10,
verify_security: bool = False):
"""Agent to predict Concept Drifts from KB using a customized
Predictor for that. This agent authenticates on XMPP server.
:param jid: Id for XMPP authentication. Ex: user@localhost
:type jid: str
:param password: Password for XMPP authentication.
:type password: str
:param predictor: Predictor for Concept Drift detection.
:type predictor: PlannerPredictor
:param executors_jid: List of executors that this planner will
send detected Concept Drifts.
:type executors_jid: Iterable[str]
:param cache_max_size: Cache list if executor is unavailable,
defaults to 10
:type cache_max_size: int, optional
:param verify_security: [description], defaults to False
:type verify_security: bool, optional
"""
self._predictor = predictor
self._executors = executors_jid
self._cache = deque([], cache_max_size)
self._sent_data = {}
self._logger = getLogger("planner")
super().__init__(jid, password, verify_security)
def __init__(self,
jid: str,
password: str,
predictor: AnalyserPredictor,
database_connection: Connection,
monitors_jid: Iterable[str],
verify_security: bool = False):
"""Agent to predict Concept Drifts using a customized
Predictor for that. This agent authenticates on XMPP server.
:param jid: Id for XMPP authentication. Ex: user@localhost
:type jid: str
:param password: Password for XMPP authentication.
:type password: str
:param predictor: Predictor for Concept Drift detection.
:type predictor: AnalyserPredictor
:param database_connection: Database connection using SQLAlchemy.
:type database_connection: Connection
:param monitors_jid: List of monitors that this analyser will
predict Concept Drifts.
:type monitors_jid: Iterable[str]
:param verify_security: Security validation with XMPP server,
defaults to False.
:type verify_security: bool, optional
"""
self._monitors = monitors_jid
self._connection = database_connection
self._predictor = predictor
self._logger = getLogger("analyser")
super().__init__(jid, password, verify_security)
def __init__(self,
db_engine: Engine,
bulk_size: int,
bulk_time: Union[int, float],
circuit_breaker: CircuitBreaker = CircuitBreaker()):
"""Connects with SQLAlchemy Engine to store and query
data for concept drift datection.
:param db_engine: SQLAlchemy Engine to use as backend
:type db_engine: Engine
:param bulk_size: Quantity of data that connection will
wait to make bulk insert
:type bulk_size: int
:param bulk_time: Time in seconds between last insert and now.
If bulk_size is not reached in bulk_time interval,
then an insert was done
:type bulk_time: Union[int, float]
:param circuit_breaker: Circuit Breaker configuration to
connect with Database, defaults to CircuitBreaker()
:type circuit_breaker: CircuitBreaker, optional
"""
self._jid = None
self._conn = db_engine
self._bulk_size = bulk_size
self._bulk_time = bulk_time
self._last_insert_time = datetime.utcnow()
self._bulk_df = pd.DataFrame()
self.get_between = circuit_breaker(self.get_between)
self._insert = circuit_breaker(self._insert)
self._logger = getLogger("connection")
class TrainPredictor(PeriodicBehaviour):
_logger = getLogger("train_predictor")
async def run(self):
"""[summary]
"""
await self.agent.predictor.fit()
self._logger.debug("Analyser model fitted")
class ReceiveNewData(CyclicBehaviour):
_logger = getLogger("receive_new_data")
async def run(self):
"""[summary]
"""
msg = await self.receive()
if msg:
self.agent.add_behaviour(StoreNewData(), Template(body=msg.body))
self._logger.debug(f"Message received {msg.body}")
class FastNotifyContacts(OneShotBehaviour):
_logger = getLogger("fast_notify_contacts")
async def run(self):
"""[summary]
"""
for contact in self.agent.available_contacts.copy():
msg = Message(to=contact, body=self.template.body)
await self.send(msg)
self._logger.debug(f"Sent {self.template.body} to all contacts")
class SendNewData(OneShotBehaviour):
_logger = getLogger("send_new_data")
async def run(self):
"""[summary]
"""
body = orjson.loads(self.template.body)
for msg in body:
if self.agent.sink.is_available():
await self.agent.sink.drain(msg)
self._logger.debug(f"Message sent to Sink {msg}")
class ReceiveNewData(CyclicBehaviour):
_logger = getLogger("receive_new_data")
async def run(self):
"""[summary]
"""
if self.agent.sink.is_available():
self.presence.set_available()
else:
self.presence.set_unavailable()
msg = await self.receive()
if msg:
self.agent.add_behaviour(SendNewData(), Template(body=msg.body))
self._logger.debug(f"Message received {msg.body}")
class WaitMonitorSubscriptions(WaitSubscriptions):
_logger = getLogger("wait_monitor_subscriptions")
def on_subscribe(self, jid: str):
"""[summary]
:param jid: [description]
:type jid: [str]
"""
self.presence.approve(jid)
self.presence.subscribe(jid)
self._logger.debug(f"Approved and subscribing to {jid}")
async def run(self):
"""[summary]
"""
self.presence.on_subscribe = self.on_subscribe
await super().run()
def __init__(self,
jid: str,
password: str,
identifier: Optional[str] = None,
verify_security: bool = False):
"""An Agent to collect data from sources and send to Analyser.
This agent authenticates on XMPP server.
:param jid: Id for XMPP authentication. Ex: user@localhost
:type jid: str
:param password: Password for XMPP authentication.
:type password: str
:param identifier: Data identification or agent jid, defaults to None
:type identifier: Optional[str], optional
:param verify_security: Security validation with XMPP server,
defaults to False.
:type verify_security: bool, optional
"""
super().__init__(jid, password, verify_security)
self._identifier = identifier if identifier else self.name
self._logger = getLogger("monitor")
def __init__(self,
jid: str,
password: str,
sink: Sink,
verify_security: bool = False):
"""Agent to send predicted drifts to Sink.
This agent authenticates on XMPP server.
:param jid: Id for XMPP authentication. Ex: user@localhost
:type jid: str
:param password: Password for XMPP authentication.
:type password: str
:param sink: Where predicted Concept Drifts will be dispatched.
:type sink: Sink
:param verify_security: Security validation with XMPP server,
defaults to False.
:type verify_security: bool, optional
"""
self._sink = sink
self._logger = getLogger("executor")
super().__init__(jid, password, verify_security)
class WaitSubscriptions(OneShotBehaviour):
_logger = getLogger("wait_subscriptions")
def on_available(self, jid: str, stanza: Presence):
"""[summary]
:param jid: [description]
:type jid: [str]
:param stanza: [description]
:type stanza: [Presence]
"""
if not jid.startswith(self.agent.jid.localpart):
self.agent.available_contacts[jid] = stanza
self._logger.debug(f"Contact added {jid}")
def on_unavailable(self, jid: str, stanza: Presence):
"""[summary]
:param jid: [description]
:type jid: [str]
:param stanza: [description]
:type stanza: [Presence]
"""
try:
del self.agent.available_contacts[jid]
self._logger.debug(f"Contact removed {jid}")
except KeyError:
pass
async def run(self):
"""[summary]
"""
self.presence.on_available = self.on_available
self.presence.on_unavailable = self.on_unavailable
self.presence.set_available()
class NotifyContacts(OneShotBehaviour):
_logger = getLogger("notify_contacts")
async def run(self):
"""[summary]
"""
for contact in self.agent.available_contacts.copy():
contact_data = self.agent.sent_data[contact]
if ((len(contact_data) > 0)
and (contact_data[-1] == id(self.agent.cache[-1]))):
continue
to_send = []
to_send_id = []
for item in self.agent.cache:
if id(item) in contact_data:
continue
to_send.append(item)
to_send_id.append(id(item))
msg = Message(to=contact, body=str(orjson.dumps(to_send), "utf-8"))
await self.send(msg)
self.agent.sent_data[contact].extend(to_send_id)
self._logger.debug(f"Sent to contact {contact} data {to_send}")
class Predict(PeriodicBehaviour):
_logger = getLogger("predict")
async def run(self):
"""[summary]
"""
predictor = self.agent.predictor
results = await predictor.predict()
has_new_data = False
for result in results:
if result.should_send:
self.agent.cache.append({
"timestamp":
datetime.utcnow().timestamp(),
"identifier":
result.identifier,
"predicted":
result.predicted
})
has_new_data = True
if has_new_data:
self.agent.add_behaviour(NotifyContacts())
self._logger.debug("Notified new predictions")
def test_should_return_child_log_with_new_level(self):
logger = getLogger("test", "DEBUG")
self.assertEqual(logger.name, "driftage.test")
self.assertEqual(logger.level, 10)
def test_should_return_child_log(self):
logger = getLogger("test")
self.assertEqual(logger.name, "driftage.test")
self.assertEqual(logger.level, 20)
class StoreNewData(OneShotBehaviour):
_logger = getLogger("store_new_data")
async def run(self):
"""[summary]
"""
msg = orjson.loads(self.template.body)
data = await self._parse(msg)
predicted = await self._predict(data)
await self._store(data, predicted)
self._logger.debug(f"Data stored on database {data}")
async def _parse(self, msg: dict) -> pd.DataFrame:
"""[summary]
:param msg: [description]
:type msg: dict
:return: [description]
:rtype: PredictionData
"""
data = msg["data"]
metadata = msg["metadata"]
return PredictionData(
data=data,
timestamp=datetime.fromtimestamp(metadata["timestamp"]),
identifier=metadata["identifier"]
)
async def _predict(self, data: PredictionData) -> bool:
"""[summary]
:param data: [description]
:type data: PredictionData
:return: [description]
:rtype: bool
"""
return await self.agent.predictor.predict(data)
async def _store(self, data: PredictionData, prediction: bool):
"""[summary]
:param data: [description]
:type data: PredictionData
:param prediction: [description]
:type prediction: bool
"""
df = pd.DataFrame(
{
table.c.driftage_jid.name: [self.agent.name],
table.c.driftage_data.name: [
str(orjson.dumps(data.data), "utf-8")],
table.c.driftage_datetime_monitored.name: [data.timestamp],
table.c.driftage_datetime_analysed.name: [datetime.utcnow()],
table.c.driftage_identifier.name: [data.identifier],
table.c.driftage_predicted.name: [prediction]
}
)
await self.agent.connection.lazy_insert(df)
