def test_get_servers_from_scope_one_server(self, mock_get_now, mock_app): mock_get_now.return_value = now s0 = Server(id='00000000-0000-0000-0000-000000000000', name='node0', port=5000, me=True, created_on=old_age) db.session.add(s0) mock_app.dm.cluster_manager.get_alive.return_value = [s0.id] quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([s0], quorum) s0.created_on = now db.session.flush() quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([s0], quorum) mock_app.dm.cluster_manager.get_alive.return_value = [] quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([s0], quorum) s0.l_ignore_on_lock = True db.session.flush() quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([], quorum)
def lock_scope(scope: Scope, servers: t.Union[t.List[Server], Server] = None, bypass: t.Union[t.List[Server], Server] = None, retries=0, delay=3, applicant=None, identity=None): if not locker_scope_enabled(scope): yield None return if servers is not None: servers = servers if is_iterable_not_string(servers) else [servers] if len(servers) == 0: servers = None servers = servers or get_servers_from_scope(scope, bypass) logger.debug( f"Requesting Lock on {scope.name} to the following servers: {[s.name for s in servers]}" ) _try = 1 applicant = lock(scope, servers=servers, applicant=applicant, retries=retries, delay=delay, identity=identity) try: yield applicant finally: unlock(scope, servers=servers, applicant=applicant, identity=identity)
def test_get_servers_from_scope_less_than_min_quorum( self, mock_get_now, mock_app): mock_get_now.return_value = dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc) me = Server(id='00000000-0000-0000-0000-000000000000', name='node0', port=5000, me=True, created_on=now) db.session.add(me) servers = [] for i in range(1, 4): s = Server(f'node{i}', port=5000, created_on=old_age) r = Route(s, cost=0) db.session.add_all([s, r]) servers.append(s) mock_app.dm.cluster_manager.get_alive.return_value = [ s.id for s in servers ] quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual(4, len(quorum)) self.assertIn(Server.get_current(), quorum)
def test_get_servers_from_scope_young_server(self, mock_get_now, mock_app): mock_get_now.return_value = now s0 = Server(id='00000000-0000-0000-0000-000000000000', name='node0', port=5000, me=True, created_on=now) s1 = Server(id='00000000-0000-0000-0000-000000000001', name='node1', port=5000, created_on=now) s2 = Server(id='00000000-0000-0000-0000-000000000002', name='node2', port=5000, created_on=now) db.session.add_all([s0, s1, s2]) mock_app.dm.cluster_manager.get_alive.return_value = [s1.id, s2.id] quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([s0, s1], quorum)
def unlock(scope: Scope, applicant, servers: t.Union[t.List[Server], t.List[Id]] = None, identity=None): """ unlocks the Locker if allowed Parameters ---------- scope Returns ------- """ if scope.ORCHESTRATION == scope and servers is None: raise ValueError('servers must be set') s = None if servers: if not isinstance(servers[0], Server): engine = db.get_engine() Session = sessionmaker(bind=engine) s = Session() servers = [s.session.query(Server).get(s) for s in servers] servers = servers or get_servers_from_scope(scope) if not servers: if s: s.close() raise RuntimeError('no server to unlock') try: lock_unlock(action='U', scope=scope, servers=servers, applicant=applicant, identity=identity) except errors.LockError as e: logger.warning(f"Unable to unlock: {e}") if s: s.close()
def test_get_servers_from_scope_more_than_min_quorum( self, mock_get_now, mock_app): mock_get_now.return_value = dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc) Server.set_initial() servers = [] for i in range(0, 7): s = Server(f'node{i}', port=5000, created_on=old_age) if i == 0: r = Route(s, cost=0) else: r = Route(s, random.choice(servers), cost=i) db.session.add_all([s, r]) servers.append(s) mock_get_now.return_value = dt.datetime(2019, 4, 2, tzinfo=dt.timezone.utc) mock_app.dm.cluster_manager.get_alive.return_value = [ s.id for s in servers ] s62 = Server(f'node72', port=5000) Route(s62, random.choice(servers), cost=6) db.session.add(s62) quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual(8, len(quorum)) self.assertNotIn(s62, quorum) self.assertIn(s, quorum) self.assertIn(Server.get_current(), quorum)
def lock(scope: Scope, servers: t.List[Server] = None, applicant=None, retries=0, delay=3, identity=None) -> UUID: """ locks the Locker if allowed Parameters ---------- scope scope that lock will affect. servers if scope set to Scope.ORCHESTRATION, applicant identifier of the lock Returns ------- Result """ if servers is not None: servers = servers if is_iterable_not_string(servers) else [servers] if len(servers) == 0: servers = None servers = servers or get_servers_from_scope(scope) if len(servers) == 0: raise errors.NoServerToLock(scope) applicant = applicant if applicant is not None else [ str(s.id) for s in servers ] _try = 1 while True: try: lock_unlock(action='L', scope=scope, servers=servers, applicant=applicant, identity=identity) except errors.LockError as e: if _try < retries: _try += 1 logger.info( f"Retrying to lock on {scope.name} in {delay} seconds") time.sleep(delay) else: error_servers = [r.server for r in e.responses] locked_servers = list( set(server for server in servers) - set(error_servers)) lock_unlock('U', scope, servers=locked_servers, applicant=applicant, identity=identity) raise e else: break return applicant