def test_redis_zombies(self):
cfg_manager = ConfigurationManager.ConfigurationManager()
cfg_manager.append_module(configuration_module)
cfg_manager._set_value(
configuration_doc.SESSION_MANAGER_DEFAULT_TIMEOUT, 1)
server = SessionManager.SessionManager(cfg_manager,
SessionType.redis, "0")
try:
server.create_session()
sessions = server.list_sessions()
self.assertEquals(1, len(sessions))
time.sleep(1)
server.gateway._lock(redis.Redis(), 'foobar')
# In redis 2.4, we can not establish a timeout lower than 1 second. So
# we basically wait 1 second and few time more to confirm that it has
# been removed.
for _ in xrange(10):
server.delete_expired_sessions()
sessions = server.list_sessions()
if len(sessions) == 0:
break
time.sleep(0.2)
self.assertEquals(0, len(sessions))
finally:
server.clear()
def __init__(self, locator, cfg_manager, session_type, session_manager,
coordinator, commands_store, finished_reservations_store):
# This is an optimization. It shouldn't be stored in the SessionManager
# since the value itself doesn't matter. The important thing is that
# each UPS doesn't lock too often the global session in the
# SessionManager.
self._latest_check = 0
self._latest_check_lock = threading.RLock()
self._locator = locator
self._cfg_manager = cfg_manager
self._session_manager = session_manager
self._commands_store = commands_store
self._finished_reservations_store = finished_reservations_store
self._coordinator = coordinator
self._set_min_time_between_checks()
self._time_module = time
pool = cfg_manager.get_value(ALIVE_USERS_SESSION_POOL,
DEFAULT_ALIVE_USERS_SESSION_POOL)
# timeout = None: don't expire! Ever!
self._users_session_manager = SessionManager.SessionManager(
cfg_manager, session_type, pool, timeout=None)
self._experiments_server_session_id = self._users_session_manager.create_session(
)
self._users_session_manager.modify_session(
self._experiments_server_session_id,
[
# session_id,
])
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs):
"""
:param coord_address: Structure that identifies a concrete machine / server [correct? type?]
:param locator: Registry through which we can get access to the ExperimentServer connectors. It
is through this registry that we can actually send requests to the Experiment Servers.
:param cfg_manager: Provides access to the experiment configuration
:param args: [unused?]
:param kwargs: [unused?]
:return:
"""
super(LaboratoryServer, self).__init__(*args, **kwargs)
session_type = cfg_manager.get_value(
WEBLAB_LABORATORY_SERVER_SESSION_TYPE,
DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_TYPE)
session_pool_id = cfg_manager.get_value(
WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID,
DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID)
self._session_manager = SessionManager.SessionManager(
cfg_manager, getattr(SessionType, session_type), session_pool_id)
self._coord_address = coord_address
self._locator = locator
self._cfg_manager = cfg_manager
# This dictionary will be used to store the ongoing and not-yet-queried
# async requests. They will be stored by session.
# TODO: Consider refactoring this.
self._async_requests = {}
self._load_assigned_experiments()
def setUp(self):
self.cfg_manager = ConfigurationManager.ConfigurationManager()
self.cfg_manager.append_module(configuration_module)
self.session_manager = SessionManager.SessionManager(
self.cfg_manager, SessionType.Memory, "foo")
def func():
self.session_manager.create_session()
self.runner = stress_util.MainRunner(func, "SessionManager")
def setUp(self):
cfg_manager = ConfigurationManager.ConfigurationManager()
cfg_manager.append_module(configuration_module)
cfg_manager._set_value(
configuration_doc.SESSION_MEMORY_GATEWAY_SERIALIZE, True)
self.memory_server1 = SessionManager.SessionManager(
cfg_manager, SessionType.Memory, "foo")
self.memory_server2 = SessionManager.SessionManager(
cfg_manager, SessionType.Memory, "bar")
self.sqlalchemy_server1 = SessionManager.SessionManager(
cfg_manager, SessionType.sqlalchemy, "foo")
self.sqlalchemy_server2 = SessionManager.SessionManager(
cfg_manager, SessionType.sqlalchemy, "bar")
if REDIS_AVAILABLE:
self.redis_server1 = SessionManager.SessionManager(
cfg_manager, SessionType.redis, "0")
self.redis_server2 = SessionManager.SessionManager(
cfg_manager, SessionType.redis, "1")
else:
global ERROR_MESSAGE_SHOWN
if not ERROR_MESSAGE_SHOWN:
print(
"redis not available. Skipping redis sessions related tests",
file=sys.stderr)
ERROR_MESSAGE_SHOWN = True
self.memory_server1.clear()
self.memory_server2.clear()
self.sqlalchemy_server1.clear()
self.sqlalchemy_server2.clear()
if REDIS_AVAILABLE:
self.redis_server1.clear()
self.redis_server2.clear()
def _create_session_manager(self):
session_type = self._cfg_manager.get_value(
WEBLAB_PROXY_SERVER_SESSION_TYPE,
DEFAULT_WEBLAB_PROXY_SERVER_SESSION_TYPE)
session_pool_id = self._cfg_manager.get_value(
WEBLAB_PROXY_SERVER_SESSION_POOL_ID,
DEFAULT_WEBLAB_PROXY_SERVER_SESSION_POOL_ID)
if session_type in SessionType.getSessionTypeValues():
return SessionManager.SessionManager(self._cfg_manager,
session_type, session_pool_id)
else:
raise ProxyErrors.NotASessionTypeError('Not a session type: %s' %
session_type)
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs):
super(VisirExperiment, self).__init__(*args, **kwargs)
self._cfg_manager = cfg_manager
self.read_config()
self._requesting_lock = threading.Lock()
# We will initialize and start it later
self.heartbeater = None
self.heartbeater_lock = threading.Lock()
self._users_counter_lock = threading.Lock()
self.users_counter = 0
# XXX It must be SessionType.Memory, since we are storing an HTTPConnection object
self._session_manager = SessionManager.SessionManager( cfg_manager, SessionType.Memory, "visir" )
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs):
super(VisirExperiment, self).__init__(*args, **kwargs)
self._cfg_manager = cfg_manager
self.read_config()
self._requesting_lock = threading.Lock()
self._users_counter_lock = threading.Lock()
self.users_counter = 0
# XXX It must be SessionType.Memory, since we are storing an HTTPConnection object
self._session_manager = SessionManager.SessionManager(
cfg_manager, SessionType.Memory, "visir")
# We initialize the heartbeater here
self.heartbeater = Heartbeater(self, self.heartbeat_period,
self._session_manager)
self.heartbeater.setDaemon(True)
self.heartbeater.setName('Heartbeater')
self.heartbeater.start()
def __init__(self, cfg_manager, map=None, map_file=None, *args, **kwargs):
"""
session_type: member of voodoo.sessions.session_type
map: voodoo.gen.coordinator.CoordinationInformation.CoordinationMap
map_file: file object
The parameter session_type must be provided; if "map" parameter is provided, the CoordinatorServer
uses this map. If map_file is provided, a new CoordinatorMap is created, and loaded from the map_file.
If no one of these two parameters is provided, a new CoordinatorMap is created, waiting for the method
"load" to be called. Finally, if both parameters are provided, an exception is raised.
"""
super(CoordinatorServer, self).__init__(*args, **kwargs)
session_type = cfg_manager.get_value(
COORDINATOR_SERVER_SESSION_TYPE,
DEFAULT_COORDINATOR_SERVER_SESSION_TYPE)
session_pool_id = cfg_manager.get_value(
COORDINATOR_SERVER_SESSION_POOL_ID,
DEFAULT_COORDINATOR_SERVER_SESSION_POOL_ID)
if session_type in SessionType.getSessionTypeValues():
self._session_manager = SessionManager.SessionManager(
cfg_manager, session_type, session_pool_id)
else:
raise CoordinatorServerErrors.NotASessionTypeError(
"Not a session_type: %s" % session_type)
if map is not None and map_file is None:
self._coordination_map_controller = CoordinationInformation.CoordinationMapController(
map)
elif map is None and map_file is not None:
self._coordination_map_controller = CoordinationInformation.CoordinationMapController(
)
self._coordination_map_controller.load(map_file)
elif map is not None and map_file is not None:
raise CoordinatorServerErrors.BothMapAndMapFileProvidedError(
"Can't provide both map_file and map to CoordinatorServer")
elif map is None and map_file is None:
raise CoordinatorServerErrors.NeitherMapNorFileProvidedError(
"Can't build the Coordination Map if neither map nor map_file fields are provided!"
)
else:
raise RuntimeError(
"This possibility should never happen -voodoo.gen.coordinator.CoordinatorServer.__init__-"
)
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs):
super(LaboratoryServer, self).__init__(*args, **kwargs)
session_type = cfg_manager.get_value(
WEBLAB_LABORATORY_SERVER_SESSION_TYPE,
DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_TYPE)
session_pool_id = cfg_manager.get_value(
WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID,
DEFAULT_WEBLAB_LABORATORY_SERVER_SESSION_POOL_ID)
self._session_manager = SessionManager.SessionManager(
cfg_manager, getattr(SessionType, session_type), session_pool_id)
self._coord_address = coord_address
self._locator = locator
self._cfg_manager = cfg_manager
# This dictionary will be used to store the ongoing and not-yet-queried
# async requests. They will be stored by session.
# TODO: Consider refactoring this.
self._async_requests = {}
self._load_assigned_experiments()
def setUp(self):
cfg_manager = ConfigurationManager.ConfigurationManager()
cfg_manager.append_module(configuration_module)
commands_store = TemporalInformationStore.CommandsTemporalInformationStore(
)
locator = UserProcessorTest.FakeLocator(None)
self.session_mgr = SessionManager.SessionManager(
cfg_manager, SessionType.Memory, "foo")
coordinator = DummyCoordinator()
self.finished_reservations_store = Queue.Queue()
self.auc = AliveUsersCollection.AliveUsersCollection(
locator, cfg_manager, SessionType.Memory, self.session_mgr,
coordinator, commands_store, self.finished_reservations_store)
self.tm = TimeModule()
self.auc._time_module = self.tm
def __init__(self, coord_address, locator, cfg_manager, dont_start = False, *args, **kwargs):
super(UserProcessingServer,self).__init__(*args, **kwargs)
log.log( UserProcessingServer, log.level.Info, "Starting Core Server...")
self._stopping = False
self._cfg_manager = cfg_manager
self.config = cfg_manager
self._locator = locator
self.core_server_url = cfg_manager.get_doc_value(configuration_doc.CORE_SERVER_URL)
if cfg_manager.get_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER, 'default') == 'default' or cfg_manager.get_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER_HUMAN, 'default') == 'default':
generated = uuid.uuid1()
msg = "Property %(property)s or %(property_human)s not configured. Please establish: %(property)s = '%(uuid)s' and %(property_human)s = 'server at university X'. Otherwise, when federating the experiment it could enter in an endless loop." % {
'property' : configuration_doc.CORE_UNIVERSAL_IDENTIFIER,
'property_human' : configuration_doc.CORE_UNIVERSAL_IDENTIFIER_HUMAN,
'uuid' : generated
}
print(msg)
print(msg, file = sys.stderr)
log.log( UserProcessingServer, log.level.Error, msg)
self.core_server_universal_id = cfg_manager.get_doc_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER)
self.core_server_universal_id_human = cfg_manager.get_doc_value(configuration_doc.CORE_UNIVERSAL_IDENTIFIER_HUMAN)
#
# Create session managers
#
session_type_str = cfg_manager.get_doc_value(configuration_doc.WEBLAB_CORE_SERVER_SESSION_TYPE)
if not hasattr(SessionType, session_type_str):
raise coreExc.NotASessionTypeError( 'Not a session type: %s' % session_type_str )
session_type = getattr(SessionType, session_type_str)
session_pool_id = cfg_manager.get_doc_value(configuration_doc.WEBLAB_CORE_SERVER_SESSION_POOL_ID)
self._session_manager = SessionManager.SessionManager( cfg_manager, session_type, session_pool_id )
reservations_session_pool_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_RESERVATIONS_SESSION_POOL_ID, "CoreServerReservations")
self._reservations_session_manager = SessionManager.SessionManager( cfg_manager, session_type, reservations_session_pool_id )
#
# Coordination
#
coordinator_implementation = cfg_manager.get_doc_value(configuration_doc.COORDINATOR_IMPL)
self._coordinator = coordinator_create(coordinator_implementation, self._locator, cfg_manager)
#
# Database and information storage managers
#
self._db_manager = DatabaseGateway(cfg_manager)
self.db = self._db_manager
cfg_manager.client = DbConfig(self.db.client_configuration)
cfg_manager.server = DbConfig(self.db.server_configuration)
self._commands_store = TemporalInformationStore.CommandsTemporalInformationStore()
self._temporal_information_retriever = TemporalInformationRetriever.TemporalInformationRetriever(cfg_manager, self._coordinator.initial_store, self._coordinator.finished_store, self._commands_store, self._coordinator.completed_store, self._db_manager)
self._temporal_information_retriever.start()
clean = cfg_manager.get('core_number') == 0
if clean:
self._location_retriever = LocationRetriever(cfg_manager, self._db_manager)
self._location_retriever.start()
else:
self._location_retriever = None
#
# Alive users
#
self._alive_users_collection = AliveUsersCollection.AliveUsersCollection(
self._locator, self._cfg_manager, session_type, self._reservations_session_manager, self._coordinator, self._commands_store, self._coordinator.finished_reservations_store)
# Login Manager
self._login_manager = LoginManager(self._db_manager, self)
#
# Initialize facade (comm) servers
#
self._server_route = cfg_manager.get_doc_value(configuration_doc.CORE_FACADE_SERVER_ROUTE)
self.flask_server = WebLabFlaskServer(self, cfg_manager)
self.babel = self.flask_server.babel
self.dont_start = cfg_manager.get_value('dont_start', dont_start)
if not self.dont_start:
self.flask_server.start()
self.app = self.flask_server.app
#
# Start checking times
#
checking_time = self._cfg_manager.get_value(CHECKING_TIME_NAME, DEFAULT_CHECKING_TIME)
timer = threading.Timer(checking_time, self._renew_checker_timer)
timer.setName(counter.next_name("ups-checker-timer"))
timer.setDaemon(True)
timer.start()
_resource_manager.add_resource(timer)
def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs):
super(UserProcessingServer,self).__init__(*args, **kwargs)
log.log( UserProcessingServer, log.level.Info, "Starting Core Server...")
self._stopping = False
self._cfg_manager = cfg_manager
self._locator = locator
if cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER, 'default') == 'default' or cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN, 'default') == 'default':
generated = uuid.uuid1()
msg = "Property %(property)s or %(property_human)s not configured. Please establish: %(property)s = '%(uuid)s' and %(property_human)s = 'server at university X'. Otherwise, when federating the experiment it could enter in an endless loop." % {
'property' : WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER,
'property_human' : WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN,
'uuid' : generated
}
print msg
print >> sys.stderr, msg
log.log( UserProcessingServer, log.level.Error, msg)
self.core_server_universal_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER, DEFAULT_WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER)
self.core_server_universal_id_human = cfg_manager.get_value(WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN, DEFAULT_WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER_HUMAN)
#
# Create session managers
#
session_type_str = cfg_manager.get_doc_value(configuration_doc.WEBLAB_CORE_SERVER_SESSION_TYPE)
if not hasattr(SessionType, session_type_str):
raise coreExc.NotASessionTypeError( 'Not a session type: %s' % session_type_str )
session_type = getattr(SessionType, session_type_str)
session_pool_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_SESSION_POOL_ID, "UserProcessingServer")
self._session_manager = SessionManager.SessionManager( cfg_manager, session_type, session_pool_id )
reservations_session_pool_id = cfg_manager.get_value(WEBLAB_CORE_SERVER_RESERVATIONS_SESSION_POOL_ID, "CoreServerReservations")
self._reservations_session_manager = SessionManager.SessionManager( cfg_manager, session_type, reservations_session_pool_id )
#
# Coordination
#
coordinator_implementation = cfg_manager.get_value(WEBLAB_CORE_SERVER_COORDINATION_IMPLEMENTATION, SQLALCHEMY)
self._coordinator = coordinator_create(coordinator_implementation, self._locator, cfg_manager)
#
# Database and information storage managers
#
self._db_manager = DatabaseManager.UserProcessingDatabaseManager(cfg_manager)
self._commands_store = TemporalInformationStore.CommandsTemporalInformationStore()
self._temporal_information_retriever = TemporalInformationRetriever.TemporalInformationRetriever(cfg_manager, self._coordinator.initial_store, self._coordinator.finished_store, self._commands_store, self._coordinator.completed_store, self._db_manager)
self._temporal_information_retriever.start()
#
# Alive users
#
self._alive_users_collection = AliveUsersCollection.AliveUsersCollection(
self._locator, self._cfg_manager, session_type, self._reservations_session_manager, self._coordinator, self._commands_store, self._coordinator.finished_reservations_store)
#
# Initialize facade (comm) servers
#
self._server_route = cfg_manager.get_value(UserProcessingFacadeServer.USER_PROCESSING_FACADE_SERVER_ROUTE, UserProcessingFacadeServer.DEFAULT_USER_PROCESSING_SERVER_ROUTE)
self._facade_servers = []
for FacadeClass in self.FACADE_SERVERS:
facade_server = FacadeClass(self, cfg_manager)
self._facade_servers.append(facade_server)
facade_server.start()
#
# Start checking times
#
self._initialize_checker_timer()
