def test_redis_zombies(self): cfg_manager = ConfigurationManager.ConfigurationManager() cfg_manager.append_module(configuration_module) cfg_manager._set_value( configuration_doc.SESSION_MANAGER_DEFAULT_TIMEOUT, 1) server = SessionManager.SessionManager(cfg_manager, SessionType.redis, "0") try: server.create_session() sessions = server.list_sessions() self.assertEquals(1, len(sessions)) time.sleep(1) server.gateway._lock(redis.Redis(), 'foobar') # In redis 2.4, we can not establish a timeout lower than 1 second. So # we basically wait 1 second and few time more to confirm that it has # been removed. for _ in xrange(10): server.delete_expired_sessions() sessions = server.list_sessions() if len(sessions) == 0: break time.sleep(0.2) self.assertEquals(0, len(sessions)) finally: server.clear()
def __init__(self, locator, cfg_manager, session_type, session_manager, coordinator, commands_store, finished_reservations_store): # This is an optimization. It shouldn't be stored in the SessionManager # since the value itself doesn't matter. The important thing is that # each UPS doesn't lock too often the global session in the # SessionManager. self._latest_check = 0 self._latest_check_lock = threading.RLock() self._locator = locator self._cfg_manager = cfg_manager self._session_manager = session_manager self._commands_store = commands_store self._finished_reservations_store = finished_reservations_store self._coordinator = coordinator self._set_min_time_between_checks() self._time_module = time pool = cfg_manager.get_value(ALIVE_USERS_SESSION_POOL, DEFAULT_ALIVE_USERS_SESSION_POOL) # timeout = None: don't expire! Ever! self._users_session_manager = SessionManager.SessionManager( cfg_manager, session_type, pool, timeout=None) self._experiments_server_session_id = self._users_session_manager.create_session( ) self._users_session_manager.modify_session( self._experiments_server_session_id, [ # session_id, ])
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs): """ :param coord_address: Structure that identifies a concrete machine / server [correct? type?] :param locator: Registry through which we can get access to the ExperimentServer connectors. It is through this registry that we can actually send requests to the Experiment Servers. :param cfg_manager: Provides access to the experiment configuration :param args: [unused?] :param kwargs: [unused?] :return: """ super(LaboratoryServer, self).__init__(*args, **kwargs) session_type = cfg_manager.get_value( WEBLAB_LABORATORY_SERVER_SESSION_TYPE, DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_TYPE) session_pool_id = cfg_manager.get_value( WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID, DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID) self._session_manager = SessionManager.SessionManager( cfg_manager, getattr(SessionType, session_type), session_pool_id) self._coord_address = coord_address self._locator = locator self._cfg_manager = cfg_manager # This dictionary will be used to store the ongoing and not-yet-queried # async requests. They will be stored by session. # TODO: Consider refactoring this. self._async_requests = {} self._load_assigned_experiments()
def setUp(self): self.cfg_manager = ConfigurationManager.ConfigurationManager() self.cfg_manager.append_module(configuration_module) self.session_manager = SessionManager.SessionManager( self.cfg_manager, SessionType.Memory, "foo") def func(): self.session_manager.create_session() self.runner = stress_util.MainRunner(func, "SessionManager")
def setUp(self): cfg_manager = ConfigurationManager.ConfigurationManager() cfg_manager.append_module(configuration_module) cfg_manager._set_value( configuration_doc.SESSION_MEMORY_GATEWAY_SERIALIZE, True) self.memory_server1 = SessionManager.SessionManager( cfg_manager, SessionType.Memory, "foo") self.memory_server2 = SessionManager.SessionManager( cfg_manager, SessionType.Memory, "bar") self.sqlalchemy_server1 = SessionManager.SessionManager( cfg_manager, SessionType.sqlalchemy, "foo") self.sqlalchemy_server2 = SessionManager.SessionManager( cfg_manager, SessionType.sqlalchemy, "bar") if REDIS_AVAILABLE: self.redis_server1 = SessionManager.SessionManager( cfg_manager, SessionType.redis, "0") self.redis_server2 = SessionManager.SessionManager( cfg_manager, SessionType.redis, "1") else: global ERROR_MESSAGE_SHOWN if not ERROR_MESSAGE_SHOWN: print( "redis not available. Skipping redis sessions related tests", file=sys.stderr) ERROR_MESSAGE_SHOWN = True self.memory_server1.clear() self.memory_server2.clear() self.sqlalchemy_server1.clear() self.sqlalchemy_server2.clear() if REDIS_AVAILABLE: self.redis_server1.clear() self.redis_server2.clear()
def _create_session_manager(self): session_type = self._cfg_manager.get_value( WEBLAB_PROXY_SERVER_SESSION_TYPE, DEFAULT_WEBLAB_PROXY_SERVER_SESSION_TYPE) session_pool_id = self._cfg_manager.get_value( WEBLAB_PROXY_SERVER_SESSION_POOL_ID, DEFAULT_WEBLAB_PROXY_SERVER_SESSION_POOL_ID) if session_type in SessionType.getSessionTypeValues(): return SessionManager.SessionManager(self._cfg_manager, session_type, session_pool_id) else: raise ProxyErrors.NotASessionTypeError('Not a session type: %s' % session_type)
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs): super(VisirExperiment, self).__init__(*args, **kwargs) self._cfg_manager = cfg_manager self.read_config() self._requesting_lock = threading.Lock() # We will initialize and start it later self.heartbeater = None self.heartbeater_lock = threading.Lock() self._users_counter_lock = threading.Lock() self.users_counter = 0 # XXX It must be SessionType.Memory, since we are storing an HTTPConnection object self._session_manager = SessionManager.SessionManager( cfg_manager, SessionType.Memory, "visir" )
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs): super(VisirExperiment, self).__init__(*args, **kwargs) self._cfg_manager = cfg_manager self.read_config() self._requesting_lock = threading.Lock() self._users_counter_lock = threading.Lock() self.users_counter = 0 # XXX It must be SessionType.Memory, since we are storing an HTTPConnection object self._session_manager = SessionManager.SessionManager( cfg_manager, SessionType.Memory, "visir") # We initialize the heartbeater here self.heartbeater = Heartbeater(self, self.heartbeat_period, self._session_manager) self.heartbeater.setDaemon(True) self.heartbeater.setName('Heartbeater') self.heartbeater.start()
def __init__(self, cfg_manager, map=None, map_file=None, *args, **kwargs): """ session_type: member of voodoo.sessions.session_type map: voodoo.gen.coordinator.CoordinationInformation.CoordinationMap map_file: file object The parameter session_type must be provided; if "map" parameter is provided, the CoordinatorServer uses this map. If map_file is provided, a new CoordinatorMap is created, and loaded from the map_file. If no one of these two parameters is provided, a new CoordinatorMap is created, waiting for the method "load" to be called. Finally, if both parameters are provided, an exception is raised. """ super(CoordinatorServer, self).__init__(*args, **kwargs) session_type = cfg_manager.get_value( COORDINATOR_SERVER_SESSION_TYPE, DEFAULT_COORDINATOR_SERVER_SESSION_TYPE) session_pool_id = cfg_manager.get_value( COORDINATOR_SERVER_SESSION_POOL_ID, DEFAULT_COORDINATOR_SERVER_SESSION_POOL_ID) if session_type in SessionType.getSessionTypeValues(): self._session_manager = SessionManager.SessionManager( cfg_manager, session_type, session_pool_id) else: raise CoordinatorServerErrors.NotASessionTypeError( "Not a session_type: %s" % session_type) if map is not None and map_file is None: self._coordination_map_controller = CoordinationInformation.CoordinationMapController( map) elif map is None and map_file is not None: self._coordination_map_controller = CoordinationInformation.CoordinationMapController( ) self._coordination_map_controller.load(map_file) elif map is not None and map_file is not None: raise CoordinatorServerErrors.BothMapAndMapFileProvidedError( "Can't provide both map_file and map to CoordinatorServer") elif map is None and map_file is None: raise CoordinatorServerErrors.NeitherMapNorFileProvidedError( "Can't build the Coordination Map if neither map nor map_file fields are provided!" ) else: raise RuntimeError( "This possibility should never happen -voodoo.gen.coordinator.CoordinatorServer.__init__-" )
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs): super(LaboratoryServer, self).__init__(*args, **kwargs) session_type = cfg_manager.get_value( WEBLAB_LABORATORY_SERVER_SESSION_TYPE, DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_TYPE) session_pool_id = cfg_manager.get_value( WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID, DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID) self._session_manager = SessionManager.SessionManager( cfg_manager, getattr(SessionType, session_type), session_pool_id) self._coord_address = coord_address self._locator = locator self._cfg_manager = cfg_manager # This dictionary will be used to store the ongoing and not-yet-queried # async requests. They will be stored by session. # TODO: Consider refactoring this. self._async_requests = {} self._load_assigned_experiments()
def setUp(self): cfg_manager = ConfigurationManager.ConfigurationManager() cfg_manager.append_module(configuration_module) commands_store = TemporalInformationStore.CommandsTemporalInformationStore( ) locator = UserProcessorTest.FakeLocator(None) self.session_mgr = SessionManager.SessionManager( cfg_manager, SessionType.Memory, "foo") coordinator = DummyCoordinator() self.finished_reservations_store = Queue.Queue() self.auc = AliveUsersCollection.AliveUsersCollection( locator, cfg_manager, SessionType.Memory, self.session_mgr, coordinator, commands_store, self.finished_reservations_store) self.tm = TimeModule() self.auc._time_module = self.tm
def __init__(self, coord_address, locator, cfg_manager, dont_start = False, *args, **kwargs): super(UserProcessingServer,self).__init__(*args, **kwargs) log.log( UserProcessingServer, log.level.Info, "Starting Core Server...") self._stopping = False self._cfg_manager = cfg_manager self.config = cfg_manager self._locator = locator self.core_server_url = cfg_manager.get_doc_value(configuration_doc.CORE_SERVER_URL) if cfg_manager.get_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER, 'default') == 'default' or cfg_manager.get_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER_HUMAN, 'default') == 'default': generated = uuid.uuid1() msg = "Property %(property)s or %(property_human)s not configured. Please establish: %(property)s = '%(uuid)s' and %(property_human)s = 'server at university X'. Otherwise, when federating the experiment it could enter in an endless loop." % { 'property' : configuration_doc.CORE_UNIVERSAL_IDENTIFIER, 'property_human' : configuration_doc.CORE_UNIVERSAL_IDENTIFIER_HUMAN, 'uuid' : generated } print(msg) print(msg, file = sys.stderr) log.log( UserProcessingServer, log.level.Error, msg) self.core_server_universal_id = cfg_manager.get_doc_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER) self.core_server_universal_id_human = cfg_manager.get_doc_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER_HUMAN) # # Create session managers # session_type_str = cfg_manager.get_doc_value(configuration_doc.WEBLAB_CORE_SERVER_SESSION_TYPE) if not hasattr(SessionType, session_type_str): raise coreExc.NotASessionTypeError( 'Not a session type: %s' % session_type_str ) session_type = getattr(SessionType, session_type_str) session_pool_id = cfg_manager.get_doc_value(configuration_doc.WEBLAB_CORE_SERVER_SESSION_POOL_ID) self._session_manager = SessionManager.SessionManager( cfg_manager, session_type, session_pool_id ) reservations_session_pool_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_RESERVATIONS_SESSION_POOL_ID, "CoreServerReservations") self._reservations_session_manager = SessionManager.SessionManager( cfg_manager, session_type, reservations_session_pool_id ) # # Coordination # coordinator_implementation = cfg_manager.get_doc_value(configuration_doc.COORDINATOR_IMPL) self._coordinator = coordinator_create(coordinator_implementation, self._locator, cfg_manager) # # Database and information storage managers # self._db_manager = DatabaseGateway(cfg_manager) self.db = self._db_manager cfg_manager.client = DbConfig(self.db.client_configuration) cfg_manager.server = DbConfig(self.db.server_configuration) self._commands_store = TemporalInformationStore.CommandsTemporalInformationStore() self._temporal_information_retriever = TemporalInformationRetriever.TemporalInformationRetriever(cfg_manager, self._coordinator.initial_store, self._coordinator.finished_store, self._commands_store, self._coordinator.completed_store, self._db_manager) self._temporal_information_retriever.start() clean = cfg_manager.get('core_number') == 0 if clean: self._location_retriever = LocationRetriever(cfg_manager, self._db_manager) self._location_retriever.start() else: self._location_retriever = None # # Alive users # self._alive_users_collection = AliveUsersCollection.AliveUsersCollection( self._locator, self._cfg_manager, session_type, self._reservations_session_manager, self._coordinator, self._commands_store, self._coordinator.finished_reservations_store) # Login Manager self._login_manager = LoginManager(self._db_manager, self) # # Initialize facade (comm) servers # self._server_route = cfg_manager.get_doc_value(configuration_doc.CORE_FACADE_SERVER_ROUTE) self.flask_server = WebLabFlaskServer(self, cfg_manager) self.babel = self.flask_server.babel self.dont_start = cfg_manager.get_value('dont_start', dont_start) if not self.dont_start: self.flask_server.start() self.app = self.flask_server.app # # Start checking times # checking_time = self._cfg_manager.get_value(CHECKING_TIME_NAME, DEFAULT_CHECKING_TIME) timer = threading.Timer(checking_time, self._renew_checker_timer) timer.setName(counter.next_name("ups-checker-timer")) timer.setDaemon(True) timer.start() _resource_manager.add_resource(timer)
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs): super(UserProcessingServer,self).__init__(*args, **kwargs) log.log( UserProcessingServer, log.level.Info, "Starting Core Server...") self._stopping = False self._cfg_manager = cfg_manager self._locator = locator if cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER, 'default') == 'default' or cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN, 'default') == 'default': generated = uuid.uuid1() msg = "Property %(property)s or %(property_human)s not configured. Please establish: %(property)s = '%(uuid)s' and %(property_human)s = 'server at university X'. Otherwise, when federating the experiment it could enter in an endless loop." % { 'property' : WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER, 'property_human' : WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN, 'uuid' : generated } print msg print >> sys.stderr, msg log.log( UserProcessingServer, log.level.Error, msg) self.core_server_universal_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER, DEFAULT_WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER) self.core_server_universal_id_human = cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN, DEFAULT_WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN) # # Create session managers # session_type_str = cfg_manager.get_doc_value(configuration_doc.WEBLAB_CORE_SERVER_SESSION_TYPE) if not hasattr(SessionType, session_type_str): raise coreExc.NotASessionTypeError( 'Not a session type: %s' % session_type_str ) session_type = getattr(SessionType, session_type_str) session_pool_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_SESSION_POOL_ID, "UserProcessingServer") self._session_manager = SessionManager.SessionManager( cfg_manager, session_type, session_pool_id ) reservations_session_pool_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_RESERVATIONS_SESSION_POOL_ID, "CoreServerReservations") self._reservations_session_manager = SessionManager.SessionManager( cfg_manager, session_type, reservations_session_pool_id ) # # Coordination # coordinator_implementation = cfg_manager.get_value(WEBLAB_CORE_SERVER_COORDINATION_IMPLEMENTATION, SQLALCHEMY) self._coordinator = coordinator_create(coordinator_implementation, self._locator, cfg_manager) # # Database and information storage managers # self._db_manager = DatabaseManager.UserProcessingDatabaseManager(cfg_manager) self._commands_store = TemporalInformationStore.CommandsTemporalInformationStore() self._temporal_information_retriever = TemporalInformationRetriever.TemporalInformationRetriever(cfg_manager, self._coordinator.initial_store, self._coordinator.finished_store, self._commands_store, self._coordinator.completed_store, self._db_manager) self._temporal_information_retriever.start() # # Alive users # self._alive_users_collection = AliveUsersCollection.AliveUsersCollection( self._locator, self._cfg_manager, session_type, self._reservations_session_manager, self._coordinator, self._commands_store, self._coordinator.finished_reservations_store) # # Initialize facade (comm) servers # self._server_route = cfg_manager.get_value(UserProcessingFacadeServer.USER_PROCESSING_FACADE_SERVER_ROUTE, UserProcessingFacadeServer.DEFAULT_USER_PROCESSING_SERVER_ROUTE) self._facade_servers = [] for FacadeClass in self.FACADE_SERVERS: facade_server = FacadeClass(self, cfg_manager) self._facade_servers.append(facade_server) facade_server.start() # # Start checking times # self._initialize_checker_timer()