def test_get_servers_from_scope_one_server(self, mock_get_now, mock_app):
mock_get_now.return_value = now
s0 = Server(id='00000000-0000-0000-0000-000000000000',
name='node0',
port=5000,
me=True,
created_on=old_age)
db.session.add(s0)
mock_app.dm.cluster_manager.get_alive.return_value = [s0.id]
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([s0], quorum)
s0.created_on = now
db.session.flush()
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([s0], quorum)
mock_app.dm.cluster_manager.get_alive.return_value = []
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([s0], quorum)
s0.l_ignore_on_lock = True
db.session.flush()
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([], quorum)
def lock_scope(scope: Scope,
servers: t.Union[t.List[Server], Server] = None,
bypass: t.Union[t.List[Server], Server] = None,
retries=0,
delay=3,
applicant=None,
identity=None):
if not locker_scope_enabled(scope):
yield None
return
if servers is not None:
servers = servers if is_iterable_not_string(servers) else [servers]
if len(servers) == 0:
servers = None
servers = servers or get_servers_from_scope(scope, bypass)
logger.debug(
f"Requesting Lock on {scope.name} to the following servers: {[s.name for s in servers]}"
)
_try = 1
applicant = lock(scope,
servers=servers,
applicant=applicant,
retries=retries,
delay=delay,
identity=identity)
try:
yield applicant
finally:
unlock(scope, servers=servers, applicant=applicant, identity=identity)
def test_get_servers_from_scope_less_than_min_quorum(
self, mock_get_now, mock_app):
mock_get_now.return_value = dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc)
me = Server(id='00000000-0000-0000-0000-000000000000',
name='node0',
port=5000,
me=True,
created_on=now)
db.session.add(me)
servers = []
for i in range(1, 4):
s = Server(f'node{i}', port=5000, created_on=old_age)
r = Route(s, cost=0)
db.session.add_all([s, r])
servers.append(s)
mock_app.dm.cluster_manager.get_alive.return_value = [
s.id for s in servers
]
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual(4, len(quorum))
self.assertIn(Server.get_current(), quorum)
def test_get_servers_from_scope_young_server(self, mock_get_now, mock_app):
mock_get_now.return_value = now
s0 = Server(id='00000000-0000-0000-0000-000000000000',
name='node0',
port=5000,
me=True,
created_on=now)
s1 = Server(id='00000000-0000-0000-0000-000000000001',
name='node1',
port=5000,
created_on=now)
s2 = Server(id='00000000-0000-0000-0000-000000000002',
name='node2',
port=5000,
created_on=now)
db.session.add_all([s0, s1, s2])
mock_app.dm.cluster_manager.get_alive.return_value = [s1.id, s2.id]
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([s0, s1], quorum)
def unlock(scope: Scope,
applicant,
servers: t.Union[t.List[Server], t.List[Id]] = None,
identity=None):
"""
unlocks the Locker if allowed
Parameters
----------
scope
Returns
-------
"""
if scope.ORCHESTRATION == scope and servers is None:
raise ValueError('servers must be set')
s = None
if servers:
if not isinstance(servers[0], Server):
engine = db.get_engine()
Session = sessionmaker(bind=engine)
s = Session()
servers = [s.session.query(Server).get(s) for s in servers]
servers = servers or get_servers_from_scope(scope)
if not servers:
if s:
s.close()
raise RuntimeError('no server to unlock')
try:
lock_unlock(action='U',
scope=scope,
servers=servers,
applicant=applicant,
identity=identity)
except errors.LockError as e:
logger.warning(f"Unable to unlock: {e}")
if s:
s.close()
def test_get_servers_from_scope_more_than_min_quorum(
self, mock_get_now, mock_app):
mock_get_now.return_value = dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc)
Server.set_initial()
servers = []
for i in range(0, 7):
s = Server(f'node{i}', port=5000, created_on=old_age)
if i == 0:
r = Route(s, cost=0)
else:
r = Route(s, random.choice(servers), cost=i)
db.session.add_all([s, r])
servers.append(s)
mock_get_now.return_value = dt.datetime(2019,
4,
2,
tzinfo=dt.timezone.utc)
mock_app.dm.cluster_manager.get_alive.return_value = [
s.id for s in servers
]
s62 = Server(f'node72', port=5000)
Route(s62, random.choice(servers), cost=6)
db.session.add(s62)
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual(8, len(quorum))
self.assertNotIn(s62, quorum)
self.assertIn(s, quorum)
self.assertIn(Server.get_current(), quorum)
def lock(scope: Scope,
servers: t.List[Server] = None,
applicant=None,
retries=0,
delay=3,
identity=None) -> UUID:
"""
locks the Locker if allowed
Parameters
----------
scope
scope that lock will affect.
servers
if scope set to Scope.ORCHESTRATION,
applicant
identifier of the lock
Returns
-------
Result
"""
if servers is not None:
servers = servers if is_iterable_not_string(servers) else [servers]
if len(servers) == 0:
servers = None
servers = servers or get_servers_from_scope(scope)
if len(servers) == 0:
raise errors.NoServerToLock(scope)
applicant = applicant if applicant is not None else [
str(s.id) for s in servers
]
_try = 1
while True:
try:
lock_unlock(action='L',
scope=scope,
servers=servers,
applicant=applicant,
identity=identity)
except errors.LockError as e:
if _try < retries:
_try += 1
logger.info(
f"Retrying to lock on {scope.name} in {delay} seconds")
time.sleep(delay)
else:
error_servers = [r.server for r in e.responses]
locked_servers = list(
set(server for server in servers) - set(error_servers))
lock_unlock('U',
scope,
servers=locked_servers,
applicant=applicant,
identity=identity)
raise e
else:
break
return applicant