def test_is_migration_in_process(self): playertracker1 = PlayerTracker() self.assertFalse(playertracker1.is_migration_in_process()) conductor = Conductor() self.assertTrue(playertracker1.is_migration_in_process()) del conductor self.assertFalse(playertracker1.is_migration_in_process())
def test_ACK_COMMAND_DISCONNECT_PGPOOL(self): reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.acknowledgement_pgpool_disconnected(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) player_tracker = PlayerTracker() self.assertTrue(player_tracker.is_pgpool_disconnected(112)) self.assertFalse(player_tracker.is_pgpool_disconnected(115))
def test_set_pgpool_disconnected(self): playertracker1 = PlayerTracker() playertracker1.set_pgpool_disconnected(123) self.assertFalse(playertracker1.is_pgpool_connected(123)) self.assertTrue(playertracker1.is_pgpool_disconnected(123)) playertracker1.set_pgpool_connected(123) self.assertTrue(playertracker1.is_pgpool_connected(123)) self.assertFalse(playertracker1.is_pgpool_disconnected(123))
def setUp(self): self.lock.acquire() self.__connection = Connection("memory://") self.__thread = ConsumerThread(self.__connection) self.__thread.start() time.sleep(1) PlayerTracker().clear() PlayerTracker().set_accept_player(True)
def test_set_master_disconnected(self): playertracker1 = PlayerTracker() playertracker1.set_replication_stopped(123) self.assertFalse(playertracker1.is_replication_started(123)) self.assertTrue(playertracker1.is_replication_stopped(123)) playertracker1.set_replication_started(123) self.assertTrue(playertracker1.is_replication_started(123)) self.assertFalse(playertracker1.is_pgpool_disconnected(123))
def test_ACK_COMMAND_FIND_PLAYER(self): reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) player_tracker = PlayerTracker() ids = player_tracker.get_player_ids(timeout=5) self.assertEqual(2, len(ids)) self.assertIn(112, ids) self.assertIn(115, ids)
def test_wait_replication_started(self): tested = False with Conductor() as conductor1: playerTracker = PlayerTracker() playerTracker.set_accept_player(True) playerTracker.add_player(222) playerTracker.set_accept_player(False) self.assertRaises(ConductorTimeoutException, conductor1.wait_replication_started, timeout=1) playerTracker.set_replication_started(222) conductor1.wait_replication_started(timeout=1) tested = True self.assertTrue(tested)
def test_ACK_COMMAND_FIND_PLAYER(self): reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) player_tracker = PlayerTracker() ids = player_tracker.get_player_ids(timeout=5) self.assertEqual(2, len(ids)) self.assertIn(112, ids) self.assertIn(115, ids)
def test_reset(self): playertracker1 = PlayerTracker() playertracker2 = PlayerTracker() ids = playertracker1.get_player_ids() self.assertEqual(1, len(ids)) playertracker1.clear() ids = playertracker1.get_player_ids() self.assertEqual(0, len(ids)) ids = playertracker2.get_player_ids() self.assertEqual(0, len(ids))
def test_ACK_COMMAND_DISCONNECT_PGPOOL(self): reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.acknowledgement_pgpool_disconnected( 112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) player_tracker = PlayerTracker() self.assertTrue(player_tracker.is_pgpool_disconnected(112)) self.assertFalse(player_tracker.is_pgpool_disconnected(115))
def test_accept_players(self): with Conductor() as conductor1: conductor1.accept_players() PlayerTracker().add_player(12345) ids = conductor1.get_player_ids() self.assertEqual(1, len(ids)) self.assertEqual(12345, ids[0])
def test_reject_players(self): tested = False with Conductor() as conductor1: conductor1.reject_players() playerTracker = PlayerTracker() self.assertRaises(PlayerDelayedRegistrationException, playerTracker.add_player, 123123) tested = True self.assertTrue(tested)
def __init__(self, locktimeout=60, replication_lag_tolerance=100, apply_lag_tolerance=100, time_lag_tolerance=100, monitor_timeout=28800): self.__player_trakcer = None self.__replication_lag_tolerance = replication_lag_tolerance self.__apply_lag_tolerance = apply_lag_tolerance self.__time_lag_tolerance = time_lag_tolerance self.__monitor_timeout = monitor_timeout if not self.__lock.acquire(timeout=locktimeout): raise ConductorTimeoutException() self.__player_trakcer = PlayerTracker() self.__player_trakcer.clear() self.__player_trakcer.set_migration_in_process(True) self.__broadcast_queue = Constants.BROADCAST_EXCHANGE
def test_send_disconnect_PGPool(self): with Conductor() as conductor1: conductor1.send_disconnect_PGPool() self.assertRaises(Empty, Test._queue.get, timeout=1) playerTracker = PlayerTracker() playerTracker.set_accept_player(True) playerTracker.add_player(222) playerTracker.set_accept_player(False) conductor1.send_disconnect_PGPool() message = Test._queue.get(timeout=5) self.assertEqual(Constants.ACK_COMMAND_DISCONNECT_PGPOOL, message.payload[Constants.MESSAGE_ACK_COMMAND])
def test_send_start_replication(self): with Conductor() as conductor1: conductor1.send_start_replication() self.assertRaises(Empty, Test._queue.get, timeout=1) playerTracker = PlayerTracker() playerTracker.set_accept_player(True) playerTracker.add_player(222) playerTracker.set_accept_player(False) conductor1.send_start_replication() message = Test._queue.get(timeout=5) self.assertEqual(Constants.ACK_COMMAND_START_REPLICATION, message.payload[Constants.MESSAGE_ACK_COMMAND])
def test_send_reset_players(self): with Conductor() as conductor1: conductor1.send_reset_players() self.assertRaises(Empty, Test._queue.get, timeout=1) playerTracker = PlayerTracker() playerTracker.set_accept_player(True) playerTracker.add_player(222) playerTracker.set_accept_player(False) conductor1.send_reset_players() message = Test._queue.get(timeout=5) self.assertEqual(Constants.ACK_COMMAND_RESET_PLAYERS, message.payload[Constants.MESSAGE_ACK_COMMAND])
def __init__(self, locktimeout=60, replication_lag_tolerance=100, apply_lag_tolerance=100, time_lag_tolerance=100, monitor_timeout=28800): self.__player_trakcer = None self.__replication_lag_tolerance = replication_lag_tolerance self.__apply_lag_tolerance = apply_lag_tolerance self.__time_lag_tolerance = time_lag_tolerance self.__monitor_timeout = monitor_timeout if not self.__lock.acquire(timeout=locktimeout): raise ConductorTimeoutException() self.__player_trakcer = PlayerTracker() self.__player_trakcer.clear() self.__player_trakcer.set_migration_in_process(True) self.__broadcast_queue = Constants.BROADCAST_EXCHANGE
def __init__(self, connection): self.connection = connection self.__player_tracker = PlayerTracker() self.__CONSUMER_COMMAND_HANDLERS = { Constants.ACK_COMMAND_FIND_PLAYER: self.__player_tracker.add_player, Constants.ACK_COMMAND_START_REPLICATION: self.__player_tracker.set_replication_started, Constants.ACK_COMMAND_CONNECT_PGPOOL: self.__player_tracker.set_pgpool_connected, Constants.ACK_COMMAND_STOP_REPLICATION: self.__player_tracker.set_replication_stopped, Constants.ACK_COMMAND_DISCONNECT_PGPOOL: self.__player_tracker.set_pgpool_disconnected }
def test__is_player_status(self): playertracker1 = PlayerTracker() self.assertFalse(playertracker1._is_player_status(123, 'test', 'abc')) playertracker1._set_player_status(123, 'test', 'abc') self.assertTrue(playertracker1._is_player_status(123, 'test', 'abc')) timeout = 3 start_time = time.time() self.assertRaises(PlayerStatusTimedoutException, playertracker1._is_player_status, 1, 'test', 'abc', timeout) end_time = time.time() self.assertTrue(end_time - start_time > timeout)
def test_ACK_COMMAND_CONNECT_MASTER(self): reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) player_tracker = PlayerTracker() self.assertFalse(player_tracker.is_replication_started(112)) self.assertFalse(player_tracker.is_replication_started(115)) reply_to_conductor.acknowledgement_master_disconnected(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) self.assertTrue(player_tracker.is_replication_stopped(112)) self.assertFalse(player_tracker.is_replication_started(115)) reply_to_conductor.acknowledgement_master_connected(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) self.assertTrue(player_tracker.is_replication_started(112)) self.assertFalse(player_tracker.is_replication_started(115))
def test_PlayerStatusLockingTimedoutException(self): Singleton._instances.clear() PlayerTracker._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, PlayerTracker, timeout=1) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1 = PlayerTracker(timeout=1) PlayerTracker._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_migration_in_process, None) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_migration_in_process, None) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_timeout, None) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_accept_player, None) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1.set_accept_player(True) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.add_player, 1) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1.set_accept_player(False) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.get_player_ids, None) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.clear) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1._set_player_status, None, None, None) self.assertRaises(RuntimeError, PlayerTracker._lock.release) playertracker1._lock.acquire() self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1._is_player_status, None, None, None) self.assertRaises(RuntimeError, PlayerTracker._lock.release)
def test_ACK_COMMAND_CONNECT_MASTER(self): reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) player_tracker = PlayerTracker() self.assertFalse(player_tracker.is_replication_started(112)) self.assertFalse(player_tracker.is_replication_started(115)) reply_to_conductor.acknowledgement_master_disconnected( 112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) self.assertTrue(player_tracker.is_replication_stopped(112)) self.assertFalse(player_tracker.is_replication_started(115)) reply_to_conductor.acknowledgement_master_connected( 112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY) time.sleep(2) self.assertTrue(player_tracker.is_replication_started(112)) self.assertFalse(player_tracker.is_replication_started(115))
def test_add_player(self): playertracker1 = PlayerTracker() playertracker2 = PlayerTracker() players = playertracker2.get_player_ids() self.assertEqual(1, len(players)) playertracker1.set_accept_player(True) self.assertRaises(PlayerAlreadyRegisteredException, playertracker1.add_player, 123) playertracker1.add_player(2) playertracker1.set_accept_player(False) players = playertracker2.get_player_ids() self.assertEqual(2, len(players)) self.assertIn(123, players) self.assertIn(2, players)
def test_get_player_ids(self): tested = False with Conductor() as conductor1: playerTracker = PlayerTracker() playerTracker.set_accept_player(True) playerTracker.add_player(123) playerTracker.add_player(222) playerTracker.add_player(333) playerTracker.add_player(444) playerTracker.set_accept_player(False) self.assertRaises(PlayerDelayedRegistrationException, playerTracker.add_player, 123123) ids = conductor1.get_player_ids() self.assertEqual(4, len(ids)) self.assertIn(123, ids) self.assertIn(222, ids) self.assertIn(333, ids) self.assertIn(444, ids) ids = playerTracker.get_player_ids(Constants.PLAYER_GROUP_A) self.assertEqual(0, len(ids)) tested = True self.assertTrue(tested)
def setUp(self): PlayerTracker().clear()
def test__set_player_status(self): playertracker1 = PlayerTracker() playertracker1._set_player_status(123, Constants.PLAYER_PGPOOL_CONNECTION_STATUS, Constants.PLAYER_CONNECTION_STATUS_DISCONNECTED) self.assertTrue(playertracker1.is_pgpool_disconnected(123)) self.assertRaises(PlayerNotRegisteredException, playertracker1._set_player_status, 1, Constants.PLAYER_PGPOOL_CONNECTION_STATUS, Constants.PLAYER_CONNECTION_STATUS_DISCONNECTED)
def test_set_timeout(self): playertracker1 = PlayerTracker() playertracker1.set_timeout(1) self.assertEqual(1, playertracker1.get_timeout()) playertracker1.set_timeout(2) self.assertEqual(2, playertracker1.get_timeout())
def test_set_accept_player(self): playertracker1 = PlayerTracker() playertracker1.clear() ids = playertracker1.get_player_ids() self.assertEqual(0, len(ids)) playertracker1.set_accept_player(True) playertracker1.add_player(1) playertracker1.add_player(2) playertracker1.set_accept_player(False) ids = playertracker1.get_player_ids() self.assertEqual(2, len(ids)) self.assertRaises(PlayerDelayedRegistrationException, playertracker1.add_player, 3) ids = playertracker1.get_player_ids() self.assertEqual(2, len(ids))
def test_singletone(self): playertracker1 = PlayerTracker() playertracker2 = PlayerTracker() self.assertTrue(playertracker1 == playertracker2)
def setUp(self): playertracker1 = PlayerTracker() playertracker1.clear() playertracker1.set_accept_player(True) playertracker1.add_player(123) playertracker1.set_accept_player(False)
def test_grouping_players(self): tested = False with Conductor() as conductor1: playerTracker = PlayerTracker() playerTracker.set_accept_player(True) playerTracker.add_player(123) playerTracker.add_player(222) playerTracker.add_player(333) playerTracker.add_player(444) playerTracker.set_accept_player(False) conductor1.grouping_players() groupA = playerTracker.get_player_ids(Constants.PLAYER_GROUP_A) self.assertEqual(2, len(groupA)) groupB = playerTracker.get_player_ids(Constants.PLAYER_GROUP_B) self.assertEqual(2, len(groupB)) tested = True self.assertTrue(tested)
def test_set_pgpool_connected_no_player_exist(self): playertracker1 = PlayerTracker() self.assertRaises(PlayerNotRegisteredException, playertracker1.set_pgpool_disconnected, 1)
class Conductor: __lock = threading.Lock() def __init__(self, locktimeout=60, replication_lag_tolerance=100, apply_lag_tolerance=100, time_lag_tolerance=100, monitor_timeout=28800): self.__player_trakcer = None self.__replication_lag_tolerance = replication_lag_tolerance self.__apply_lag_tolerance = apply_lag_tolerance self.__time_lag_tolerance = time_lag_tolerance self.__monitor_timeout = monitor_timeout if not self.__lock.acquire(timeout=locktimeout): raise ConductorTimeoutException() self.__player_trakcer = PlayerTracker() self.__player_trakcer.clear() self.__player_trakcer.set_migration_in_process(True) self.__broadcast_queue = Constants.BROADCAST_EXCHANGE def __enter__(self): return self def __exit__(self, exc_type, value, tb): if self.__player_trakcer: self.__player_trakcer.set_migration_in_process(False) if self.__lock.locked(): self.__lock.release() def __del__(self): if self.__player_trakcer: self.__player_trakcer.set_migration_in_process(False) if self.__lock.locked(): self.__lock.release() def send_reset_players(self): group_ids = self.__player_trakcer.get_player_ids() if group_ids: player_task.apply_async((Constants.COMMAND_RESET_PLAYERS, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable self.__log(Constants.COMMAND_RESET_PLAYERS, None, group_ids) for my_id in group_ids: self.__player_trakcer.reset_player(my_id) else: logger.debug('Command[' + Constants.COMMAND_RESET_PLAYERS + '] was not sent because there is no registered players') def accept_players(self): self.__player_trakcer.set_accept_player(True) def reject_players(self): self.__player_trakcer.set_accept_player(False) def find_players(self): player_task.apply_async((Constants.COMMAND_REGISTER_PLAYER, None), exchange=self.__broadcast_queue) # @UndefinedVariable self.__log(Constants.COMMAND_REGISTER_PLAYER, None, None) def get_player_ids(self): return self.__player_trakcer.get_player_ids() def grouping_players(self): player_ids = self.__player_trakcer.get_player_ids() if player_ids: for idx in range(len(player_ids)): # set group A for "0" or group B for "1" self.__player_trakcer.set_player_group(player_ids[idx], Constants.PLAYER_GROUP_A if idx % 2 == 0 else Constants.PLAYER_GROUP_B) def send_disconnect_PGPool(self, player_group=None): group_ids = self.__player_trakcer.get_player_ids(player_group=player_group) player_task.apply_async((Constants.COMMAND_DISCONNECT_PGPOOL, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable self.__log(Constants.COMMAND_DISCONNECT_PGPOOL, player_group, group_ids) def send_connect_PGPool(self, player_group=None): group_ids = self.__player_trakcer.get_player_ids(player_group=player_group) player_task.apply_async((Constants.COMMAND_CONNECT_PGPOOL, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable self.__log(Constants.COMMAND_CONNECT_PGPOOL, player_group, group_ids) def send_stop_replication(self, player_group=None): group_ids = self.__player_trakcer.get_player_ids(player_group=player_group) player_task.apply_async((Constants.COMMAND_STOP_REPLICATION, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable self.__log(Constants.COMMAND_STOP_REPLICATION, player_group, group_ids) def send_start_replication(self, player_group=None): group_ids = self.__player_trakcer.get_player_ids(player_group=player_group) player_task.apply_async((Constants.COMMAND_START_REPLICATION, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable self.__log(Constants.COMMAND_START_REPLICATION, player_group, group_ids) def migrate(self, tenant=None): return start_migrate_daily_delta(tenant=tenant) def wait_PGPool_disconnected(self, player_group=None, timeout=30): self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_pgpool_disconnected) def wait_PGPool_connected(self, player_group=None, timeout=30): self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_pgpool_connected) def wait_replication_stopped(self, player_group=None, timeout=30): self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_replication_stopped) def wait_replication_started(self, player_group=None, timeout=30): self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_replication_started) def monitor_replication_status(self, player_group=None): group_ids = self.__player_trakcer.get_player_ids(player_group=player_group) replication_monitor(group_ids, replication_lag_tolerance=self.__replication_lag_tolerance, apply_lag_tolerance=self.__apply_lag_tolerance, time_lag_tolerance=self.__time_lag_tolerance, timeout=self.__monitor_timeout) def __wait_for_status(self, player_group, timeout, func): group_ids = self.__player_trakcer.get_player_ids(player_group=player_group) start_time = time.time() for node_id in group_ids: while not func(node_id): if time.time() - start_time > timeout: raise ConductorTimeoutException(func.__name__ + ' timeout') time.sleep(1) if group_ids: logger.debug('function[' + func.__name__ + '] returned [' + ', '.join(str(x) for x in group_ids) + ']') else: logger.debug('function[' + func.__name__ + '] returned [None]') @staticmethod def __log(command, player_group, group_ids): if group_ids is None: group_ids = [] logger.debug('Sent command[' + command + '] to group name[' + (player_group if player_group else 'None') + '] ids[' + ', '.join(str(x) for x in group_ids) + ']')
def test_get_player_ids(self): playertracker1 = PlayerTracker() ids = playertracker1.get_player_ids() self.assertEqual(1, len(ids)) playertracker1.set_accept_player(True) playertracker1.add_player(1) playertracker1.set_accept_player(False) ids = playertracker1.get_player_ids('A') self.assertEqual(0, len(ids)) playertracker1.set_player_group(123, 'A') ids = playertracker1.get_player_ids() self.assertEqual(2, len(ids)) ids = playertracker1.get_player_ids('A') self.assertEqual(1, len(ids)) ids = playertracker1.get_player_ids('B') self.assertEqual(0, len(ids))
def test_is_pgpool_connected_no_player_exist(self): playertracker1 = PlayerTracker() self.assertRaises(PlayerStatusTimedoutException, playertracker1.is_pgpool_disconnected, 1, 1)