def test_is_migration_in_process(self):
playertracker1 = PlayerTracker()
self.assertFalse(playertracker1.is_migration_in_process())
conductor = Conductor()
self.assertTrue(playertracker1.is_migration_in_process())
del conductor
self.assertFalse(playertracker1.is_migration_in_process())
def test_ACK_COMMAND_DISCONNECT_PGPOOL(self):
reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.acknowledgement_pgpool_disconnected(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
player_tracker = PlayerTracker()
self.assertTrue(player_tracker.is_pgpool_disconnected(112))
self.assertFalse(player_tracker.is_pgpool_disconnected(115))
def test_set_pgpool_disconnected(self):
playertracker1 = PlayerTracker()
playertracker1.set_pgpool_disconnected(123)
self.assertFalse(playertracker1.is_pgpool_connected(123))
self.assertTrue(playertracker1.is_pgpool_disconnected(123))
playertracker1.set_pgpool_connected(123)
self.assertTrue(playertracker1.is_pgpool_connected(123))
self.assertFalse(playertracker1.is_pgpool_disconnected(123))
def setUp(self):
self.lock.acquire()
self.__connection = Connection("memory://")
self.__thread = ConsumerThread(self.__connection)
self.__thread.start()
time.sleep(1)
PlayerTracker().clear()
PlayerTracker().set_accept_player(True)
def test_set_master_disconnected(self):
playertracker1 = PlayerTracker()
playertracker1.set_replication_stopped(123)
self.assertFalse(playertracker1.is_replication_started(123))
self.assertTrue(playertracker1.is_replication_stopped(123))
playertracker1.set_replication_started(123)
self.assertTrue(playertracker1.is_replication_started(123))
self.assertFalse(playertracker1.is_pgpool_disconnected(123))
def test_ACK_COMMAND_FIND_PLAYER(self):
reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
player_tracker = PlayerTracker()
ids = player_tracker.get_player_ids(timeout=5)
self.assertEqual(2, len(ids))
self.assertIn(112, ids)
self.assertIn(115, ids)
def test_wait_replication_started(self):
tested = False
with Conductor() as conductor1:
playerTracker = PlayerTracker()
playerTracker.set_accept_player(True)
playerTracker.add_player(222)
playerTracker.set_accept_player(False)
self.assertRaises(ConductorTimeoutException, conductor1.wait_replication_started, timeout=1)
playerTracker.set_replication_started(222)
conductor1.wait_replication_started(timeout=1)
tested = True
self.assertTrue(tested)
def test_ACK_COMMAND_FIND_PLAYER(self):
reply_to_conductor.register_player(112, self.__connection,
conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.register_player(115, self.__connection,
conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
player_tracker = PlayerTracker()
ids = player_tracker.get_player_ids(timeout=5)
self.assertEqual(2, len(ids))
self.assertIn(112, ids)
self.assertIn(115, ids)
def test_reset(self):
playertracker1 = PlayerTracker()
playertracker2 = PlayerTracker()
ids = playertracker1.get_player_ids()
self.assertEqual(1, len(ids))
playertracker1.clear()
ids = playertracker1.get_player_ids()
self.assertEqual(0, len(ids))
ids = playertracker2.get_player_ids()
self.assertEqual(0, len(ids))
def test_ACK_COMMAND_DISCONNECT_PGPOOL(self):
reply_to_conductor.register_player(112, self.__connection,
conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.register_player(115, self.__connection,
conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.acknowledgement_pgpool_disconnected(
112, self.__connection, conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
player_tracker = PlayerTracker()
self.assertTrue(player_tracker.is_pgpool_disconnected(112))
self.assertFalse(player_tracker.is_pgpool_disconnected(115))
def test_accept_players(self):
with Conductor() as conductor1:
conductor1.accept_players()
PlayerTracker().add_player(12345)
ids = conductor1.get_player_ids()
self.assertEqual(1, len(ids))
self.assertEqual(12345, ids[0])
def test_reject_players(self):
tested = False
with Conductor() as conductor1:
conductor1.reject_players()
playerTracker = PlayerTracker()
self.assertRaises(PlayerDelayedRegistrationException, playerTracker.add_player, 123123)
tested = True
self.assertTrue(tested)
def __init__(self,
locktimeout=60,
replication_lag_tolerance=100,
apply_lag_tolerance=100,
time_lag_tolerance=100,
monitor_timeout=28800):
self.__player_trakcer = None
self.__replication_lag_tolerance = replication_lag_tolerance
self.__apply_lag_tolerance = apply_lag_tolerance
self.__time_lag_tolerance = time_lag_tolerance
self.__monitor_timeout = monitor_timeout
if not self.__lock.acquire(timeout=locktimeout):
raise ConductorTimeoutException()
self.__player_trakcer = PlayerTracker()
self.__player_trakcer.clear()
self.__player_trakcer.set_migration_in_process(True)
self.__broadcast_queue = Constants.BROADCAST_EXCHANGE
def test_send_disconnect_PGPool(self):
with Conductor() as conductor1:
conductor1.send_disconnect_PGPool()
self.assertRaises(Empty, Test._queue.get, timeout=1)
playerTracker = PlayerTracker()
playerTracker.set_accept_player(True)
playerTracker.add_player(222)
playerTracker.set_accept_player(False)
conductor1.send_disconnect_PGPool()
message = Test._queue.get(timeout=5)
self.assertEqual(Constants.ACK_COMMAND_DISCONNECT_PGPOOL, message.payload[Constants.MESSAGE_ACK_COMMAND])
def test_send_start_replication(self):
with Conductor() as conductor1:
conductor1.send_start_replication()
self.assertRaises(Empty, Test._queue.get, timeout=1)
playerTracker = PlayerTracker()
playerTracker.set_accept_player(True)
playerTracker.add_player(222)
playerTracker.set_accept_player(False)
conductor1.send_start_replication()
message = Test._queue.get(timeout=5)
self.assertEqual(Constants.ACK_COMMAND_START_REPLICATION, message.payload[Constants.MESSAGE_ACK_COMMAND])
def test_send_reset_players(self):
with Conductor() as conductor1:
conductor1.send_reset_players()
self.assertRaises(Empty, Test._queue.get, timeout=1)
playerTracker = PlayerTracker()
playerTracker.set_accept_player(True)
playerTracker.add_player(222)
playerTracker.set_accept_player(False)
conductor1.send_reset_players()
message = Test._queue.get(timeout=5)
self.assertEqual(Constants.ACK_COMMAND_RESET_PLAYERS, message.payload[Constants.MESSAGE_ACK_COMMAND])
def __init__(self, locktimeout=60, replication_lag_tolerance=100, apply_lag_tolerance=100, time_lag_tolerance=100, monitor_timeout=28800):
self.__player_trakcer = None
self.__replication_lag_tolerance = replication_lag_tolerance
self.__apply_lag_tolerance = apply_lag_tolerance
self.__time_lag_tolerance = time_lag_tolerance
self.__monitor_timeout = monitor_timeout
if not self.__lock.acquire(timeout=locktimeout):
raise ConductorTimeoutException()
self.__player_trakcer = PlayerTracker()
self.__player_trakcer.clear()
self.__player_trakcer.set_migration_in_process(True)
self.__broadcast_queue = Constants.BROADCAST_EXCHANGE
def __init__(self, connection):
self.connection = connection
self.__player_tracker = PlayerTracker()
self.__CONSUMER_COMMAND_HANDLERS = {
Constants.ACK_COMMAND_FIND_PLAYER:
self.__player_tracker.add_player,
Constants.ACK_COMMAND_START_REPLICATION:
self.__player_tracker.set_replication_started,
Constants.ACK_COMMAND_CONNECT_PGPOOL:
self.__player_tracker.set_pgpool_connected,
Constants.ACK_COMMAND_STOP_REPLICATION:
self.__player_tracker.set_replication_stopped,
Constants.ACK_COMMAND_DISCONNECT_PGPOOL:
self.__player_tracker.set_pgpool_disconnected
}
def test__is_player_status(self):
playertracker1 = PlayerTracker()
self.assertFalse(playertracker1._is_player_status(123, 'test', 'abc'))
playertracker1._set_player_status(123, 'test', 'abc')
self.assertTrue(playertracker1._is_player_status(123, 'test', 'abc'))
timeout = 3
start_time = time.time()
self.assertRaises(PlayerStatusTimedoutException, playertracker1._is_player_status, 1, 'test', 'abc', timeout)
end_time = time.time()
self.assertTrue(end_time - start_time > timeout)
def test_ACK_COMMAND_CONNECT_MASTER(self):
reply_to_conductor.register_player(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.register_player(115, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
player_tracker = PlayerTracker()
self.assertFalse(player_tracker.is_replication_started(112))
self.assertFalse(player_tracker.is_replication_started(115))
reply_to_conductor.acknowledgement_master_disconnected(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
self.assertTrue(player_tracker.is_replication_stopped(112))
self.assertFalse(player_tracker.is_replication_started(115))
reply_to_conductor.acknowledgement_master_connected(112, self.__connection, conductor.exchange, Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
self.assertTrue(player_tracker.is_replication_started(112))
self.assertFalse(player_tracker.is_replication_started(115))
def test_PlayerStatusLockingTimedoutException(self):
Singleton._instances.clear()
PlayerTracker._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, PlayerTracker, timeout=1)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1 = PlayerTracker(timeout=1)
PlayerTracker._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_migration_in_process, None)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_migration_in_process, None)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_timeout, None)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.set_accept_player, None)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1.set_accept_player(True)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.add_player, 1)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1.set_accept_player(False)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.get_player_ids, None)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1.clear)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1._set_player_status, None, None, None)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
playertracker1._lock.acquire()
self.assertRaises(PlayerStatusLockingTimedoutException, playertracker1._is_player_status, None, None, None)
self.assertRaises(RuntimeError, PlayerTracker._lock.release)
def test_ACK_COMMAND_CONNECT_MASTER(self):
reply_to_conductor.register_player(112, self.__connection,
conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
reply_to_conductor.register_player(115, self.__connection,
conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
player_tracker = PlayerTracker()
self.assertFalse(player_tracker.is_replication_started(112))
self.assertFalse(player_tracker.is_replication_started(115))
reply_to_conductor.acknowledgement_master_disconnected(
112, self.__connection, conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
self.assertTrue(player_tracker.is_replication_stopped(112))
self.assertFalse(player_tracker.is_replication_started(115))
reply_to_conductor.acknowledgement_master_connected(
112, self.__connection, conductor.exchange,
Constants.CONDUCTOR_ROUTING_KEY)
time.sleep(2)
self.assertTrue(player_tracker.is_replication_started(112))
self.assertFalse(player_tracker.is_replication_started(115))
def test_add_player(self):
playertracker1 = PlayerTracker()
playertracker2 = PlayerTracker()
players = playertracker2.get_player_ids()
self.assertEqual(1, len(players))
playertracker1.set_accept_player(True)
self.assertRaises(PlayerAlreadyRegisteredException, playertracker1.add_player, 123)
playertracker1.add_player(2)
playertracker1.set_accept_player(False)
players = playertracker2.get_player_ids()
self.assertEqual(2, len(players))
self.assertIn(123, players)
self.assertIn(2, players)
def test_get_player_ids(self):
tested = False
with Conductor() as conductor1:
playerTracker = PlayerTracker()
playerTracker.set_accept_player(True)
playerTracker.add_player(123)
playerTracker.add_player(222)
playerTracker.add_player(333)
playerTracker.add_player(444)
playerTracker.set_accept_player(False)
self.assertRaises(PlayerDelayedRegistrationException, playerTracker.add_player, 123123)
ids = conductor1.get_player_ids()
self.assertEqual(4, len(ids))
self.assertIn(123, ids)
self.assertIn(222, ids)
self.assertIn(333, ids)
self.assertIn(444, ids)
ids = playerTracker.get_player_ids(Constants.PLAYER_GROUP_A)
self.assertEqual(0, len(ids))
tested = True
self.assertTrue(tested)
def setUp(self):
PlayerTracker().clear()
def test__set_player_status(self):
playertracker1 = PlayerTracker()
playertracker1._set_player_status(123, Constants.PLAYER_PGPOOL_CONNECTION_STATUS, Constants.PLAYER_CONNECTION_STATUS_DISCONNECTED)
self.assertTrue(playertracker1.is_pgpool_disconnected(123))
self.assertRaises(PlayerNotRegisteredException, playertracker1._set_player_status, 1, Constants.PLAYER_PGPOOL_CONNECTION_STATUS, Constants.PLAYER_CONNECTION_STATUS_DISCONNECTED)
def test_set_timeout(self):
playertracker1 = PlayerTracker()
playertracker1.set_timeout(1)
self.assertEqual(1, playertracker1.get_timeout())
playertracker1.set_timeout(2)
self.assertEqual(2, playertracker1.get_timeout())
def test_set_accept_player(self):
playertracker1 = PlayerTracker()
playertracker1.clear()
ids = playertracker1.get_player_ids()
self.assertEqual(0, len(ids))
playertracker1.set_accept_player(True)
playertracker1.add_player(1)
playertracker1.add_player(2)
playertracker1.set_accept_player(False)
ids = playertracker1.get_player_ids()
self.assertEqual(2, len(ids))
self.assertRaises(PlayerDelayedRegistrationException, playertracker1.add_player, 3)
ids = playertracker1.get_player_ids()
self.assertEqual(2, len(ids))
def test_singletone(self):
playertracker1 = PlayerTracker()
playertracker2 = PlayerTracker()
self.assertTrue(playertracker1 == playertracker2)
def setUp(self):
playertracker1 = PlayerTracker()
playertracker1.clear()
playertracker1.set_accept_player(True)
playertracker1.add_player(123)
playertracker1.set_accept_player(False)
def test_grouping_players(self):
tested = False
with Conductor() as conductor1:
playerTracker = PlayerTracker()
playerTracker.set_accept_player(True)
playerTracker.add_player(123)
playerTracker.add_player(222)
playerTracker.add_player(333)
playerTracker.add_player(444)
playerTracker.set_accept_player(False)
conductor1.grouping_players()
groupA = playerTracker.get_player_ids(Constants.PLAYER_GROUP_A)
self.assertEqual(2, len(groupA))
groupB = playerTracker.get_player_ids(Constants.PLAYER_GROUP_B)
self.assertEqual(2, len(groupB))
tested = True
self.assertTrue(tested)
def test_set_pgpool_connected_no_player_exist(self):
playertracker1 = PlayerTracker()
self.assertRaises(PlayerNotRegisteredException, playertracker1.set_pgpool_disconnected, 1)
class Conductor:
__lock = threading.Lock()
def __init__(self, locktimeout=60, replication_lag_tolerance=100, apply_lag_tolerance=100, time_lag_tolerance=100, monitor_timeout=28800):
self.__player_trakcer = None
self.__replication_lag_tolerance = replication_lag_tolerance
self.__apply_lag_tolerance = apply_lag_tolerance
self.__time_lag_tolerance = time_lag_tolerance
self.__monitor_timeout = monitor_timeout
if not self.__lock.acquire(timeout=locktimeout):
raise ConductorTimeoutException()
self.__player_trakcer = PlayerTracker()
self.__player_trakcer.clear()
self.__player_trakcer.set_migration_in_process(True)
self.__broadcast_queue = Constants.BROADCAST_EXCHANGE
def __enter__(self):
return self
def __exit__(self, exc_type, value, tb):
if self.__player_trakcer:
self.__player_trakcer.set_migration_in_process(False)
if self.__lock.locked():
self.__lock.release()
def __del__(self):
if self.__player_trakcer:
self.__player_trakcer.set_migration_in_process(False)
if self.__lock.locked():
self.__lock.release()
def send_reset_players(self):
group_ids = self.__player_trakcer.get_player_ids()
if group_ids:
player_task.apply_async((Constants.COMMAND_RESET_PLAYERS, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable
self.__log(Constants.COMMAND_RESET_PLAYERS, None, group_ids)
for my_id in group_ids:
self.__player_trakcer.reset_player(my_id)
else:
logger.debug('Command[' + Constants.COMMAND_RESET_PLAYERS + '] was not sent because there is no registered players')
def accept_players(self):
self.__player_trakcer.set_accept_player(True)
def reject_players(self):
self.__player_trakcer.set_accept_player(False)
def find_players(self):
player_task.apply_async((Constants.COMMAND_REGISTER_PLAYER, None), exchange=self.__broadcast_queue) # @UndefinedVariable
self.__log(Constants.COMMAND_REGISTER_PLAYER, None, None)
def get_player_ids(self):
return self.__player_trakcer.get_player_ids()
def grouping_players(self):
player_ids = self.__player_trakcer.get_player_ids()
if player_ids:
for idx in range(len(player_ids)):
# set group A for "0" or group B for "1"
self.__player_trakcer.set_player_group(player_ids[idx], Constants.PLAYER_GROUP_A if idx % 2 == 0 else Constants.PLAYER_GROUP_B)
def send_disconnect_PGPool(self, player_group=None):
group_ids = self.__player_trakcer.get_player_ids(player_group=player_group)
player_task.apply_async((Constants.COMMAND_DISCONNECT_PGPOOL, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable
self.__log(Constants.COMMAND_DISCONNECT_PGPOOL, player_group, group_ids)
def send_connect_PGPool(self, player_group=None):
group_ids = self.__player_trakcer.get_player_ids(player_group=player_group)
player_task.apply_async((Constants.COMMAND_CONNECT_PGPOOL, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable
self.__log(Constants.COMMAND_CONNECT_PGPOOL, player_group, group_ids)
def send_stop_replication(self, player_group=None):
group_ids = self.__player_trakcer.get_player_ids(player_group=player_group)
player_task.apply_async((Constants.COMMAND_STOP_REPLICATION, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable
self.__log(Constants.COMMAND_STOP_REPLICATION, player_group, group_ids)
def send_start_replication(self, player_group=None):
group_ids = self.__player_trakcer.get_player_ids(player_group=player_group)
player_task.apply_async((Constants.COMMAND_START_REPLICATION, group_ids), exchange=self.__broadcast_queue) # @UndefinedVariable
self.__log(Constants.COMMAND_START_REPLICATION, player_group, group_ids)
def migrate(self, tenant=None):
return start_migrate_daily_delta(tenant=tenant)
def wait_PGPool_disconnected(self, player_group=None, timeout=30):
self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_pgpool_disconnected)
def wait_PGPool_connected(self, player_group=None, timeout=30):
self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_pgpool_connected)
def wait_replication_stopped(self, player_group=None, timeout=30):
self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_replication_stopped)
def wait_replication_started(self, player_group=None, timeout=30):
self.__wait_for_status(player_group, timeout, self.__player_trakcer.is_replication_started)
def monitor_replication_status(self, player_group=None):
group_ids = self.__player_trakcer.get_player_ids(player_group=player_group)
replication_monitor(group_ids, replication_lag_tolerance=self.__replication_lag_tolerance, apply_lag_tolerance=self.__apply_lag_tolerance, time_lag_tolerance=self.__time_lag_tolerance, timeout=self.__monitor_timeout)
def __wait_for_status(self, player_group, timeout, func):
group_ids = self.__player_trakcer.get_player_ids(player_group=player_group)
start_time = time.time()
for node_id in group_ids:
while not func(node_id):
if time.time() - start_time > timeout:
raise ConductorTimeoutException(func.__name__ + ' timeout')
time.sleep(1)
if group_ids:
logger.debug('function[' + func.__name__ + '] returned [' + ', '.join(str(x) for x in group_ids) + ']')
else:
logger.debug('function[' + func.__name__ + '] returned [None]')
@staticmethod
def __log(command, player_group, group_ids):
if group_ids is None:
group_ids = []
logger.debug('Sent command[' + command + '] to group name[' + (player_group if player_group else 'None') + '] ids[' + ', '.join(str(x) for x in group_ids) + ']')
def test_get_player_ids(self):
playertracker1 = PlayerTracker()
ids = playertracker1.get_player_ids()
self.assertEqual(1, len(ids))
playertracker1.set_accept_player(True)
playertracker1.add_player(1)
playertracker1.set_accept_player(False)
ids = playertracker1.get_player_ids('A')
self.assertEqual(0, len(ids))
playertracker1.set_player_group(123, 'A')
ids = playertracker1.get_player_ids()
self.assertEqual(2, len(ids))
ids = playertracker1.get_player_ids('A')
self.assertEqual(1, len(ids))
ids = playertracker1.get_player_ids('B')
self.assertEqual(0, len(ids))
def test_is_pgpool_connected_no_player_exist(self):
playertracker1 = PlayerTracker()
self.assertRaises(PlayerStatusTimedoutException, playertracker1.is_pgpool_disconnected, 1, 1)
