def test_ping_cycle(self): scenarios = [[[SimpleCapability(str(CapabilityRegister.AGENT))], 1], [[], 1], [[SimpleCapability(str(CapabilityRegister.POOL))], 0], [[SimpleCapability("MadeUpCapability")], 0]] for scenario in scenarios: reqd_cap, expected_notification = scenario logging.info( "\n\nTesting Ping Cycle with capabilities [{}]\n\n".format( str(reqd_cap))) sa = SimpleAgent( 'test agent', start_state=State.S0, end_state=State.S1, capacity=1, task_consumption_policy=GreedyTaskConsumptionPolicy(), trace=True) time.sleep(.5) ping_srcsink = DummySrcSink("PingSrcSink", ping_topic=sa.topic) ping_workref = ping_srcsink.send_ping( required_capabilities=reqd_cap) time.sleep(1) self.assertEqual(expected_notification, len(ping_srcsink.ping_notifications)) if expected_notification > 0: self.assertEqual( ping_workref.id, ping_srcsink.ping_notifications[0].sender_work_ref.id) pub.unsubAll() return
def test_get_with_capabilities_bad_params(self): test_address_book = AddressBook() with self.assertRaises(ValueError): test_address_book.get_with_capabilities(required_capabilities=[ SimpleCapability("NotImportantForThisTest") ], match_threshold=-0.1) with self.assertRaises(ValueError): test_address_book.get_with_capabilities(required_capabilities=[ SimpleCapability("NotImportantForThisTest") ], match_threshold=1.1) with self.assertRaises(ValueError): test_address_book.get_with_capabilities(required_capabilities=[ SimpleCapability("NotImportantForThisTest") ], match_threshold=.5, n=0) with self.assertRaises(ValueError): test_address_book.get_with_capabilities(required_capabilities=[ SimpleCapability("NotImportantForThisTest") ], match_threshold=.5, n=-1) return
def test_ping_cycle(self): _ = SimpleEther("TestEther1") scenarios = [[[SimpleCapability(str(CapabilityRegister.POOL))], 1], [[], 1], [[SimpleCapability(str(CapabilityRegister.AGENT))], 0], [[SimpleCapability("MadeUpCapability")], 0]] for scenario in scenarios: reqd_cap, expected_notification = scenario logging.info( "\n\nTesting Ping Cycle with capabilities [{}]\n\n".format( str(reqd_cap))) ping_srcsink = DummySrcSink("PingSrcSink") task_pool = SimpleTaskPool('Task Pool 1') time.sleep(.25) ping_workref = ping_srcsink.send_ping( required_capabilities=reqd_cap) time.sleep(1) self.assertEqual(expected_notification, len(ping_srcsink.ping_notifications)) if expected_notification > 0: self.assertEqual( ping_workref.id, ping_srcsink.ping_notifications[0].sender_work_ref.id) pub.unsubAll() del task_pool del ping_srcsink return
def test_equality_same_type(self): capability_name_one = "DummyCapability1" capability_name_two = "DummyCapability2" test_capability_1 = SimpleCapability(capability_name_one) test_capability_2 = SimpleCapability(capability_name_one) test_capability_3 = SimpleCapability(capability_name_two) self.assertEqual(True, test_capability_1 == test_capability_2) self.assertEqual(True, test_capability_1 != test_capability_3) return
def run_ether_do_pub(self, scenario: List): """ We publish a SrcSink to the back-plane and verify that all other ethers respond with their address. """ reqd_cap, expected = scenario ether_tx1 = SimpleEther("TestEtherTx1") # We will publish to private topic and check it replicates ether_rx1 = SimpleEther("TestEtherRx1") # We will see if it gets the replicated ping. ether_rx2 = SimpleEther("TestEtherRx2") # We will see if it gets the replicated ping. ethers = [ether_tx1, ether_rx1, ether_rx2] # Force in some SrcSinks with capabilities via protected methods just for testing ds1 = DummySrcSink(name="DS1", capability=SimpleCapability(capability_name=self.capability_1)) ds2 = DummySrcSink(name="DS2", capability=SimpleCapability(capability_name=self.capability_2)) ether_rx1._update_addressbook(srcsink=ds1) ether_rx2._update_addressbook(srcsink=ds2) ping = SimpleSrcSinkPing(sender_srcsink=ether_tx1, required_capabilities=reqd_cap) # Pub to Private pub.sendMessage(topicName=Ether.back_plane_topic(), notification=ping) # Publish direct to Ether private topic time.sleep(1) # Wait for 1 sec to ensure the activity time triggers. for ether in ethers: ether.stop() # The sender of the ping request should have all the addresses on the ether ether = ether_tx1 logging.info("Checking {}".format(ether.name)) srcsink_topics = list(x.topic for x in ether.get_addressbook()) if expected == 3: self.assertEqual(expected, len(srcsink_topics)) # We expect all topics self.assertTrue(ether_rx1.topic in srcsink_topics) self.assertTrue(ether_rx2.topic in srcsink_topics) self.assertTrue(ds1.topic in srcsink_topics) # The SrcSink with reqd capability else: self.assertEqual(expected, len(srcsink_topics)) # We expect all topics + the tx self.assertTrue(ether_rx1.topic in srcsink_topics) # The topic should be in the set self.assertTrue(ether_rx2.topic in srcsink_topics) # The topic should be in the set self.assertTrue(ether_tx1.topic in srcsink_topics) # The topic should be in the set self.assertTrue(ds1.topic in srcsink_topics) # The SrcSink with reqd capability self.assertTrue(ds2.topic in srcsink_topics) # The SrcSink with reqd capability for ether in ethers: del ether del ds1 del ds2 return
def test_ping_single_srcsink(self): """ Test all the methods of injecting a SrcSinkPing into a single Ether entity. """ # Topic should be registered even if the ether does not have a matching ping capabilities required_capabilities = [TestEther.NO_CAPABILITIES_REQUIRED, [SimpleCapability("ArbitraryCapability")]] for reqd_cap in required_capabilities: for i in range(3): srcsink = DummySrcSink("DummySrcSink-2") ether = SimpleEther("TestEther1") ping = SimpleSrcSinkPing(sender_srcsink=srcsink, required_capabilities=reqd_cap) if i == 0: ether(ping) # Invoke as callable elif i == 1: pub.sendMessage(topicName=ether.topic, notification=ping) # Publish direct to Ether private topic else: pub.sendMessage(topicName=Ether.back_plane_topic(), notification=ping) # Publish to back plane topic srcsink_topics = list(x.topic for x in ether.get_addressbook()) self.assertEqual(1, len(srcsink_topics)) # We expect a single topic only self.assertTrue(srcsink.topic in srcsink_topics) # The topic should be in the set recorded by the ether return
def test_taskpool_capabilities(self): task_pool = SimpleTaskPool('Task Pool 1') self.assertEqual( float(1), Capability.equivalence_factor( [SimpleCapability(CapabilityRegister.POOL.name)], task_pool.capabilities)) return
def _get_capabilities() -> List[Capability]: """ The capabilities of this Agent :return: List of Capabilities """ return [ SimpleCapability(capability_name=str(CapabilityRegister.AGENT)) ]
def __init__(self, ether_name: str): """ """ self._name = ether_name self._unique_topic = self._create_topic_and_subscription() super().__init__(ether_name) self._lock = threading.Lock() self._capabilities = [SimpleCapability(str(CapabilityRegister.ETHER))] self._ping_factor_threshold = 1.0 return
def _activity_manage_presence(self, current_activity_interval: float) -> float: """ Ensure that we are known on the ether & our address book has the name of at least one local pool in it. :param current_activity_interval: The current delay in seconds before activity is re-triggered. :return: The new delay in seconds before the activity is re-triggered. """ self._trace_log_update("_activity_manage_presence", current_activity_interval) back_off_reset = False if self._get_recent_ether_address() is None: logging.info( "{} not linked to Ether - sending discovery Ping".format( self.name)) back_off_reset = True pub.sendMessage(topicName=Ether.back_plane_topic(), notification=SimpleSrcSinkPing( sender_srcsink=self, required_capabilities=[ SimpleCapability( str(CapabilityRegister.ETHER)) ])) else: # We have an Ether address so we can now ping the Ether for a task pool address. if self._get_recent_pool_address() is None: logging.info( "{} Missing local Pool address - sending discovery Ping to Ether" .format(self.name)) back_off_reset = True pub.sendMessage(topicName=Ether.back_plane_topic(), notification=SimpleSrcSinkPing( sender_srcsink=self, required_capabilities=[ SimpleCapability( str(CapabilityRegister.POOL)) ])) return NotificationHandler.back_off( reset=back_off_reset, curr_interval=current_activity_interval, min_interval=Agent.PRS_TIMER, max_interval=Agent.PRD_TIMER_MAX)
def _get_recent_ether_address(self) -> SrcSink: """ Get a recent Ether address from the AddressBook. If there is no recent Ether then return None :return: Ether SrcSink or None """ ss = self._address_book.get_with_capabilities(required_capabilities=[ SimpleCapability(str(CapabilityRegister.ETHER)) ], max_age_in_seconds=60, n=1) if ss is not None: ss = ss[0] return ss
def test_basic_capability(self): test_agent = SimpleAgent( 'agent {}'.format(1), start_state=State.S0, end_state=State.S1, capacity=1, task_consumption_policy=GreedyTaskConsumptionPolicy(), trace=True) time.sleep(1) # Time for Agent discovery to settle. self.assertEqual( float(1), Capability.equivalence_factor( [SimpleCapability(CapabilityRegister.AGENT.name)], test_agent.capabilities)) return
def __init__(self, name: str, capability: SimpleCapability = SimpleCapability( capability_name='DummySrcSink'), ping_topic: str = Ether.ETHER_BACK_PLANE_TOPIC): # SrcSink - Standard boot-strap & protected members # self._name = name self._topic = UniqueTopic().topic() self._address_book = AddressBook() super().__init__() self._capabilities = [capability] self._lock = threading.Lock() self._handler = NotificationHandler(object_to_be_handler_for=self, throw_unhandled=True) self._handler.register_handler(self._do_srcsink_ping, SrcSinkPing) self._handler.register_handler(self._do_srcsink_ping_notification, SrcSinkPingNotification) self._handler.register_handler(self._do_notification, TaskNotification) self._handler.register_handler(self._do_work, WorkNotificationDo) self._handler.register_handler(self._do_work_finalise, WorkNotificationFinalise) # Public Members just for Unit Test Asserts # self.pings = list() self.ping_notifications = list() self.task_notification = list() self.work_notification = list() self.work_finalise = list() self._ping_topic = ping_topic # The topic to issue ping's on # Get connected ! self.setup_subscriptions() return
def _do_manage_presence(self, current_interval: float) -> float: """ Ensure that we are known on the ether. :param current_interval: The current activity interval :return: The optionally revised interval before the action is invoked again """ back_off_reset = False if self._get_recent_ether_address() is None: logging.info( "{} not linked to Ether - sending discovery Ping".format( self.name)) back_off_reset = True pub.sendMessage(topicName=Ether.back_plane_topic(), notification=SimpleSrcSinkPing( sender_srcsink=self, required_capabilities=[ SimpleCapability( str(CapabilityRegister.ETHER)) ])) return NotificationHandler.back_off( reset=back_off_reset, curr_interval=current_interval, min_interval=TaskPool.PRS_TIMER, max_interval=TaskPool.PRS_TIMER_MAX)
class SimpleTaskPool(TaskPool): _POOL_CAPABILITY = SimpleCapability(str(CapabilityRegister.POOL)) def __init__(self, name: str, pool_capabilities: List[Capability] = None): super().__init__(name) self._task_pool = dict() self._pool_lock = threading.RLock() self._len = 0 self._name = name self._ping_factor_threshold = float(1) self._capabilities = list() self.set_pool_capabilities(pool_capabilities) return def __del__(self): """ Set running False, which will stop timer resets etc. """ super().__del__() def terminate_all(self) -> None: """ Terminate all activity, locks, threads, timers etc """ self.__del__() return @property def topic(self) -> str: """ The unique topic name that SrcSink listens on for activity specific to it. :return: The unique SrcSink listen topic name """ return self._unique_topic @property def name(self) -> str: """ The name of the task pool :return: The name of the task pool as string """ return self._name def _put_task(self, work_notification: WorkNotificationDo) -> None: """ Add a task to the task pool which will cause it to be advertised via the relevant topic unless the task is in it's terminal state in which case it is sent back to its ultimate source :param work_notification: The task to be added """ if work_notification.task.state == work_notification.task.process_end_state( ): logging.warning( "{} expected finalised rather than do as task complete & in end state" .format(work_notification.source.name)) work_notification_finalise = SimpleWorkNotificationFinalise.finalise_factory( work_notification) logging.info("{} converted {} to finalised & processed".format( self.name, work_notification_finalise.work_ref.id)) pub.sendMessage(topicName=self.topic, notification=work_notification_finalise) else: with self._pool_lock: self._task_pool[work_notification.work_ref.id] = [ work_notification.work_ref, work_notification ] self._len += 1 return def _get_task(self, work_request: WorkRequest) -> None: """ Send the requested task to the consumer if the task has not already been sent to a consumer :param work_request: The details of the task and the consumer """ logging.info("{} received request for work ref {} from {}".format( self.name, work_request.work_ref.id, work_request.originator.name)) to_pub = None with self._pool_lock: ref_id = work_request.work_ref.id if ref_id in self._task_pool: ref, work = self._task_pool[ref_id] del self._task_pool[ref_id] self._len -= 1 to_pub = [ work_request.originator.topic, SimpleWorkNotificationDo( unique_work_ref=work_request.work_ref, task=work.task, originator=work.originator, source=self) ] if to_pub is not None: topic, notification = to_pub logging.info("{} sent task {} to {}".format( self.name, notification.work_ref.id, notification.originator.name)) pub.sendMessage(topicName=topic, notification=notification) else: logging.info("{} had NO task {} to send to {}".format( self.name, work_request.work_ref.id, work_request.originator.name)) return def _do_pub(self, current_interval: float) -> float: """ Check for any pending tasks and advertise or re-advertise them on the relevant topic :param current_interval: The current activity interval :return: The optionally revised interval before the action is invoked again """ to_pub = list() back_off_reset = False with self._pool_lock: for ref in self._task_pool.keys(): work_ref, work = self._task_pool[ref] topic = self.topic_for_capability(work.task.state) stn = SimpleTaskNotification(unique_work_ref=work_ref, task_meta=SimpleTaskMetaData( work.task.id), originator=self) to_pub.append([work_ref, topic, stn, work.task.id]) for pub_event in to_pub: back_off_reset = True ref, topic, notification, task_id = pub_event logging.info("{} stored & advertised task {} on {} = {}".format( self.name, task_id, topic, ref.id)) pub.sendMessage(topicName=topic, notification=notification) return NotificationHandler.back_off( reset=back_off_reset, curr_interval=current_interval, min_interval=TaskPool.PRS_TIMER, max_interval=TaskPool.PRS_TIMER_MAX) def _do_manage_presence(self, current_interval: float) -> float: """ Ensure that we are known on the ether. :param current_interval: The current activity interval :return: The optionally revised interval before the action is invoked again """ back_off_reset = False if self._get_recent_ether_address() is None: logging.info( "{} not linked to Ether - sending discovery Ping".format( self.name)) back_off_reset = True pub.sendMessage(topicName=Ether.back_plane_topic(), notification=SimpleSrcSinkPing( sender_srcsink=self, required_capabilities=[ SimpleCapability( str(CapabilityRegister.ETHER)) ])) return NotificationHandler.back_off( reset=back_off_reset, curr_interval=current_interval, min_interval=TaskPool.PRS_TIMER, max_interval=TaskPool.PRS_TIMER_MAX) def _do_srcsink_ping_notification( self, ping_notification: SrcSinkPingNotification) -> None: """ Handle a ping response from a srcsink :param: The srcsink notification """ logging.info("{} RX ping response for {}".format( self.name, ping_notification.src_sink.name)) if ping_notification.src_sink.topic != self.topic: # Don't count ping response from our self. for srcsink in ping_notification.responder_address_book: self._update_addressbook(srcsink=srcsink) return def _do_srcsink_ping(self, ping_request: SrcSinkPing) -> None: """ Handle a ping response from a srcsink :param: The srcsink notification """ logging.info("{} RX ping request for {}".format( self.name, ping_request.sender_srcsink.name)) # Don't count pings from our self. if ping_request.sender_srcsink.topic != self.topic: # Note the sender is alive self._update_addressbook(ping_request.sender_srcsink) if Capability.equivalence_factor( ping_request.required_capabilities, self.capabilities) >= self._ping_factor_threshold: pub.sendMessage(topicName=ping_request.sender_srcsink.topic, notification=SimpleSrcSinkNotification( responder_srcsink=self, address_book=[self], sender_workref=ping_request.work_ref)) return def _do_work_finalise( self, work_notification_final: WorkNotificationFinalise) -> None: """ Take receipt of the given completed work item that was initiated from this agent and do any final processing. """ logging.info( "{} Rx Finalised task {} from source {} in state {}".format( self.name, work_notification_final.work_ref.id, work_notification_final.originator.name, work_notification_final.task.state)) # Send to Originator for closure pub.sendMessage(topicName=work_notification_final.originator.topic, notification=work_notification_final) return def set_pool_capabilities(self, additional_capabilities: List[Capability] = None ) -> None: """ Set the given capabilities for the Agent and add the base capabilities that all Agents have. :param additional_capabilities: Optional capabilities to add to the base capabilities """ with self._pool_lock: if self._POOL_CAPABILITY not in self._capabilities: self._capabilities.append(self._POOL_CAPABILITY) if additional_capabilities is not None: for c in additional_capabilities: if c not in self._capabilities: self._capabilities.append(c) return def __str__(self) -> str: """ String dump of the current pool state :return: A string representation of the task pool. """ # Use locks so we see a consistent view of the task_pool # s = "Task Pool [{}]\n".format(self._name) with self._pool_lock: for topic in self._task_pool.keys(): s += " Topic ({})\n".format(topic) pool, lock = self._task_pool[topic] with lock: for task in pool: s += " Task <{}>\n".format(str(task)) return s def __repr__(self): return self.__str__() def __len__(self): """ The number of tasks currently in the pool :return: The number of tasks in the pool """ return self._len
def test_get_with_capabilities_partial_matches(self): test_address_book = AddressBook() cap1 = SimpleCapability("Cap1") cap2 = SimpleCapability("Cap2") cap3 = SimpleCapability("Cap3") cap4 = SimpleCapability("Cap4") capx = SimpleCapability("Cap?") ss1 = DummySrcSink("DummySrcSink-1") ss1.capabilities = [cap1] ss2 = DummySrcSink("DummySrcSink-2") ss2.capabilities = [cap1, cap2] ss3 = DummySrcSink("DummySrcSink-3") ss3.capabilities = [cap1, cap2, cap3] ss4 = DummySrcSink("DummySrcSink-4") ss4.capabilities = [cap2, cap4] test_address_book.update(ss1) time.sleep(.01) test_address_book.update(ss2) time.sleep(.01) test_address_book.update(ss3) time.sleep(.01) test_address_book.update(ss4) scenarios = [ [1, [cap1, capx], 1.0, [None], 1], [2, [cap1, cap2, cap3], 1.0, [ss3], 1], [3, [cap1, cap2], 1.0, [ss3], 1], # As ss-3 also has Cap1, Cap2 and was created last [4, [capx], 0.0, [ss4], 1], # Zero => match any so will return last one created [5, [capx], 0.3142, [None], 1], [6, [capx], 1.0, [None], 1], [7, [cap1], 1.0, [ss3], 1], [8, [cap1, cap2], 1.0, [ss3], 1], [9, [cap1, cap2], 0.5, [ss4], 1], [10, [cap1], 1.0, [ss3, ss2], 2], [11, [cap1], 1.0, [ss3, ss2, ss1], 3], [12, [cap1], 1.0, [ss3, ss2, ss1], 4], [13, [cap1], 0.0, [ss4, ss3, ss2, ss1], 4], [14, [cap1], 1.0, [ss3, ss2, ss1], 5], [15, [cap1], 1.0, [ss3, ss2, ss1], 500], [16, [cap4, capx], 1.0, [None], 10], [17, [cap4, capx], 0.5, [ss4], 10], [18, [cap3, cap4, capx], 1.0, [None], 10], [19, [cap3, cap4, capx], 0.33, [ss4, ss3], 10] ] for scenario in scenarios: case_num, caps, threshold, expected, n = scenario logging.info( "test_get_with_capabilities_partial_matches case {}".format( case_num)) res = test_address_book.get_with_capabilities( required_capabilities=caps, match_threshold=threshold, n=n) if res is None: self.assertEqual(expected[0], res) self.assertEqual(len(expected), 1) else: self.assertEqual(len(expected), len(res)) # Should have same num results. for i in range(len(expected)): # Order dependent equality. self.assertEqual(expected[i], res[i]) logging.info("case {} passed OK".format(case_num)) return
def test_ether_capability(self): ether = SimpleEther("TestEther1") self.assertEqual(float(1), Capability.equivalence_factor([SimpleCapability(CapabilityRegister.ETHER.name)], ether.capabilities)) return
def test_equiv_degree_no_given(self): """ Test the equivalency factor calc. """ cap1 = SimpleCapability("Cap_one") cap2 = SimpleCapability("Cap_two") cap3 = SimpleCapability("Cap_three") cap4 = SimpleCapability("Cap_four") cap5 = SimpleCapability("Cap_five") scenarios = [[None, None, float(1)], [None, [], float(1)], [None, [cap1], float(1)], [None, [cap1, cap2, cap3, cap4, cap5], float(1)], [[], [cap1], float(1)], [[], [cap1, cap2, cap3, cap4, cap5], float(1)], [[cap1], None, float(0)], [[cap1, cap2, cap3, cap4, cap5], None, float(0)], [[cap1], [], float(0)], [[cap1, cap2, cap3, cap4, cap5], [], float(0)], [[cap1], [cap1], float(1 / 1)], [[cap1, cap2], [cap1], float(1 / 2)], [[cap1, cap2, cap3], [cap1], float(1 / 3)], [[cap1, cap2, cap3, cap4], [cap1], float(1 / 4)], [[cap1, cap2, cap3, cap4, cap5], [cap1], float(1 / 5)], [[cap1], [cap1, cap2], float(1)], [[cap2], [cap1, cap2, cap3], float(1)], [[cap3], [cap1, cap2, cap3, cap4], float(1)], [[cap4], [cap1, cap2, cap3, cap4, cap5], float(1)], [[cap1], [cap1, cap5], float(1 / 1)], [[cap1, cap2], [cap1, cap4, cap5], float(1 / 2)], [[cap1, cap2, cap3], [cap1, cap4, cap5], float(1 / 3)], [[cap1, cap2, cap3, cap4], [cap1, cap5], float(1 / 4)], [[cap1, cap2, cap3, cap4, cap5], [cap2], float(1 / 5)], [[cap1, cap2, cap3, cap4, cap5], [cap3], float(1 / 5)], [[cap1, cap2, cap3, cap4, cap5], [cap4], float(1 / 5)], [[cap1, cap2, cap3, cap4, cap5], [cap5], float(1 / 5)]] for scenario in scenarios: reqd, given, factor = scenario iter_count = 1 if reqd is not None: if len(reqd) > 1: iter_count *= len(reqd) if given is not None: if len(given) > 1: iter_count *= len(given) for _ in range(iter_count): if reqd is not None: np.random.shuffle(reqd) if given is not None: np.random.shuffle(given) logging.info("Required [{}] Given [{}] expected {}".format( reqd, given, factor)) self.assertEqual( factor, Capability.equivalence_factor(required_capabilities=reqd, given_capabilities=given)) return
def test_equality_diff_type(self): capability_name_one = "DummyCapability1" test_capability_1 = SimpleCapability(capability_name_one) self.assertEqual(True, test_capability_1 != capability_name_one) self.assertEqual(True, test_capability_1 != int(1)) return
def test_ether_with_capabilities(self): self.run_ether_do_pub([[SimpleCapability(self.capability_1)], 3]) return
def test_simple(self): capability_name = "DummyCapability1" test_capability = SimpleCapability(capability_name) self.assertEqual(capability_name, test_capability.value()) self.assertEqual(capability_name, str(test_capability)) return