def test_reserve_experiment(self):
db_sess_id = DatabaseSession.ValidDatabaseSessionId('student2', "student")
sess_id, _ = self.ups.do_reserve_session(db_sess_id)
exp_id = ExperimentId('this does not experiment','this neither')
self.assertRaises(
coreExc.UnknownExperimentIdError,
self.ups.reserve_experiment,
sess_id, exp_id, "{}", "{}", ClientAddress.ClientAddress("127.0.0.1")
)
exp_id = ExperimentId('ud-dummy','Dummy experiments')
lab_sess_id = SessionId.SessionId("lab_session_id")
self.lab_mock.reserve_experiment(exp_id, "{}")
self.mocker.result(lab_sess_id)
self.mocker.count(0, 1)
self.lab_mock.resolve_experiment_address(lab_sess_id)
self.mocker.result(CoordAddress.CoordAddress.translate_address('foo:bar@machine'))
self.mocker.count(0, 1)
self.mocker.replay()
reservation = self.ups.reserve_experiment(
sess_id, exp_id, "{}", "{}",
ClientAddress.ClientAddress("127.0.0.1")
)
self.assertTrue( isinstance(reservation,Reservation.Reservation))
self.ups.logout(sess_id)
def setUp(self):
# Clean the global registry of servers
GLOBAL_REGISTRY.clear()
CONSUMER_CONFIG_PATH = self.FEDERATED_DEPLOYMENTS + '/consumer/'
PROVIDER1_CONFIG_PATH = self.FEDERATED_DEPLOYMENTS + '/provider1/'
PROVIDER2_CONFIG_PATH = self.FEDERATED_DEPLOYMENTS + '/provider2/'
self.consumer_handler = load_dir(CONSUMER_CONFIG_PATH).load_process(
'consumer_machine', 'main_instance')
self.provider1_handler = load_dir(PROVIDER1_CONFIG_PATH).load_process(
'provider1_machine', 'main_instance')
self.provider2_handler = load_dir(PROVIDER2_CONFIG_PATH).load_process(
'provider2_machine', 'main_instance')
self.consumer_client = WebLabDeustoClient(
"http://127.0.0.1:%s/weblab/" % 18345)
self.provider1_client = WebLabDeustoClient(
"http://127.0.0.1:%s/weblab/" % 28345)
self.provider2_client = WebLabDeustoClient(
"http://127.0.0.1:%s/weblab/" % 38345)
# dummy1: deployed in consumer, provider1, provider2
self.dummy1 = ExperimentId("dummy1", "Dummy experiments")
# dummy2: deployed in consumer
self.dummy2 = ExperimentId("dummy2", "Dummy experiments")
# dummy3: deployed in provider1 as "dummy3_with_other_name"
self.dummy3 = ExperimentId("dummy3", "Dummy experiments")
# dummy4: deployed in provider2
self.dummy4 = ExperimentId("dummy4", "Dummy experiments")
def get_experiment_id(self, reservation_id):
reservation_data = self.get_reservation_data(reservation_id)
if reservation_data is None:
raise CoordExc.ExpiredSessionError(
"Expired reservation: no experiment id found for that reservation (%s)"
% reservation_id)
return ExperimentId.parse(reservation_data[EXPERIMENT_TYPE])
def test_reserve_experiment_waiting_confirmation(self):
status = self.processor.reserve_experiment(
ExperimentId('ud-dummy', 'Dummy experiments'), "{}", "{}",
"127.0.0.1", 'uuid')
self.assertTrue(
isinstance(status,
WebLabSchedulingStatus.WaitingConfirmationQueueStatus))
def test_confirm_experiment(self):
lab_session_id = SessionId.SessionId("samplesession_id")
mock_laboratory = self.mocker.mock()
mock_laboratory.reserve_experiment(ExperimentInstanceId('inst1','exp1','cat1'), '"sample initial data"', mocker.ANY)
self.mocker.result((lab_session_id, None, 'server:inst@mach'))
self.mock_locator.real_mock = self.mocker.mock()
self.mock_locator.real_mock.get_server_from_coordaddress(
self.coord_address,
coord_addr(self.lab_address),
ServerType.Laboratory,
'all'
)
self.mocker.result((mock_laboratory,))
self.mocker.count(min=1,max=None)
self.mocker.replay()
status, reservation1_id = self.coordinator.reserve_experiment(ExperimentId('exp1','cat1'), 30, 5, True, 'sample initial data', DEFAULT_REQUEST_INFO, {})
now = datetime.datetime.fromtimestamp(int(time.time())) # Remove milliseconds as MySQL do
self.coordinator.confirmer._confirm_handler.join()
self.assertEquals( None, self.confirmer._confirm_handler.raised_exc )
status = self.coordinator.get_reservation_status(reservation1_id)
expected_status = WSS.LocalReservedStatus(reservation1_id, CoordAddress.CoordAddress.translate_address(self.lab_address), lab_session_id, 30, '{}', now, now, True, 30, 'http://www.weblab.deusto.es/weblab/client/adfas')
self.assertTrue(hasattr(status, 'timestamp_before'), "Unexpected status. Expected\n %s\n, but the obtained does not have timestamp_before:\n %s\n" % (status, expected_status))
self.assertTrue(status.timestamp_before >= now and status.timestamp_before <= now + datetime.timedelta(seconds=10),
"Unexpected status due to timestamp_before: %s; expected something like %s" % (status, expected_status))
self.assertTrue(status.timestamp_after >= now and status.timestamp_after <= now + datetime.timedelta(seconds=10),
"Unexpected status due to timestamp_after: %s; expected something like %s" % (status, expected_status))
status.timestamp_before = now
status.timestamp_after = now
self.assertEquals( expected_status, status )
def test_list_sessions(self):
exp_id1 = ExperimentId("exp1","cat1")
exp_id2 = ExperimentId("exp2","cat1")
sessions = self.reservations_manager.list_sessions(exp_id1)
self.assertEquals(0, len(sessions))
reservation1 = self.reservations_manager.create(exp_id1, "{}", REQUEST_INFO)
reservation2 = self.reservations_manager.create(exp_id1, "{}", REQUEST_INFO)
self.reservations_manager.create(exp_id2, "{}", REQUEST_INFO)
sessions = self.reservations_manager.list_sessions(exp_id1)
self.assertEquals(2, len(sessions))
self.assertTrue(reservation1 in sessions)
self.assertTrue(reservation2 in sessions)
def list_laboratories_addresses(self):
client = self._redis_maker()
laboratory_addresses = {
# laboratory_coord_address : {
# experiment_instance_id : resource_instance
# }
}
for experiment_type in client.smembers(WEBLAB_EXPERIMENT_TYPES):
experiment_id = ExperimentId.parse(experiment_type)
experiment_instance_names = client.smembers(
WEBLAB_EXPERIMENT_INSTANCES % experiment_type)
for experiment_instance_name in experiment_instance_names:
experiment_instance_id = ExperimentInstanceId(
experiment_instance_name, experiment_id.exp_name,
experiment_id.cat_name)
weblab_experiment_instance = WEBLAB_EXPERIMENT_INSTANCE % (
experiment_type, experiment_instance_name)
laboratory_address = client.hget(weblab_experiment_instance,
LAB_COORD)
resource_str = client.hget(weblab_experiment_instance,
RESOURCE_INST)
resource = Resource.parse(resource_str)
current = laboratory_addresses.get(laboratory_address, {})
current[experiment_instance_id] = resource
laboratory_addresses[laboratory_address] = current
return laboratory_addresses
def test_reserve_experiment_not_found(self):
self.coordinator._clean()
self.assertRaises(coreExc.NoAvailableExperimentFoundError,
self.processor.reserve_experiment,
ExperimentId('ud-dummy', 'Dummy experiments'), "{}",
"{}", "127.0.0.1", 'uuid')
def test_get_reservation_info(self):
db_sess_id = DatabaseSession.ValidDatabaseSessionId('student2', "student")
sess_id, _ = self.ups.do_reserve_session(db_sess_id)
exp_id = ExperimentId('ud-dummy','Dummy experiments')
lab_sess_id = SessionId.SessionId("lab_session_id")
self.lab_mock.reserve_experiment(exp_id, "{}")
self.mocker.result(lab_sess_id)
self.mocker.count(0, 1)
self.lab_mock.resolve_experiment_address(lab_sess_id)
self.mocker.result(CoordAddress.CoordAddress.translate_address('foo:bar@machine'))
self.mocker.count(0, 1)
self.mocker.replay()
reservation = self.ups.reserve_experiment(
sess_id, exp_id, "{}", "{}",
ClientAddress.ClientAddress("127.0.0.1")
)
reservation_info = self.ups.get_reservation_info(reservation.reservation_id)
self.assertEquals('ud-dummy', reservation_info.exp_name)
self.assertEquals('Dummy experiments', reservation_info.cat_name)
self.ups.logout(sess_id)
def process_Submit(self):
lab_server_id, experiment_specification, _, _ = serializer.parse_submit_request(
request.data)
ilab_request = {'operation': 'submit', 'payload': experiment_specification}
reservation_status = weblab_api.api.reserve_experiment(
ExperimentId(lab_server_id, 'iLab experiments'),
json.dumps(ilab_request), '{}')
weblab_api.ctx.other_cookies = {
'weblab_reservation_id': reservation_status.reservation_id.id
}
return """<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<soap:Body>
<SubmitResponse xmlns="http://ilab.mit.edu">
<SubmitResult>
<vReport>
<accepted>true</accepted>
<errorMessage />
<estRuntime>8</estRuntime>
</vReport>
<experimentID>61</experimentID>
<minTimeToLive>0</minTimeToLive>
<wait>
<effectiveQueueLength>%s</effectiveQueueLength>
<estWait>5</estWait>
</wait>
</SubmitResult>
</SubmitResponse>
</soap:Body>
</soap:Envelope>""" % reservation_status.position
def reserve_experiment(self, reservation_id, experiment_id, time, priority, initialization_in_accounting, client_initial_data, request_info):
server_uuids = list(request_info.get(SERVER_UUIDS, []))
server_uuids.append((self.core_server_uuid, self.core_server_uuid_human))
consumer_data = {
'time_allowed' : time,
'priority' : priority,
'initialization_in_accounting' : initialization_in_accounting,
'external_user' : request_info.get('username', ''),
SERVER_UUIDS : server_uuids,
}
for forwarded_key in FORWARDED_KEYS:
if forwarded_key in request_info:
consumer_data[forwarded_key] = request_info[forwarded_key]
# TODO: identifier of the server
login_client = self._create_login_client()
session_id = login_client.login(self.username, self.password)
client = self._create_client(login_client.get_cookies())
serialized_client_initial_data = json.dumps(client_initial_data)
serialized_consumer_data = json.dumps(consumer_data)
# If the administrator has mapped that this experiment_id is other, take that other. Otherwide, take the same one
requested_experiment_id_str = self.experiments_map.get(experiment_id.to_weblab_str(), experiment_id.to_weblab_str())
requested_experiment_id = ExperimentId.parse(requested_experiment_id_str)
external_reservation = client.reserve_experiment(session_id, requested_experiment_id, serialized_client_initial_data, serialized_consumer_data)
if external_reservation.is_null():
return None
remote_reservation_id = external_reservation.reservation_id.id
log.log(ExternalWebLabDeustoScheduler, log.level.Warning, "Local reservation_id %s is linked to remote reservation %s" % (reservation_id, remote_reservation_id))
cookies = client.get_cookies()
serialized_cookies = pickle.dumps(cookies)
redis_client = self.redis_maker()
pipeline = redis_client.pipeline()
pipeline.hset(self.external_weblabdeusto_reservations, reservation_id, json.dumps({
'remote_reservation_id' : remote_reservation_id,
'cookies' : serialized_cookies,
'start_time' : time_mod.time(),
}))
external_weblabdeusto_pending_results = self.EXTERNAL_WEBLABDEUSTO_PENDING_RESULTS % (self.resource_type_name, self.core_server_route)
pipeline.hset(external_weblabdeusto_pending_results, reservation_id, json.dumps({
'remote_reservation_id' : remote_reservation_id,
'username' : request_info.get('username',''),
'serialized_request_info' : pickle.dumps(request_info),
'experiment_id_str' : experiment_id.to_weblab_str(),
}))
pipeline.execute()
reservation_status = self._convert_reservation_to_status(external_reservation, reservation_id, remote_reservation_id)
return reservation_status
def test_get_experiment_status__one_reservation(self):
category = "Dummy experiments"
experiment = "ud-dummy"
status, reservation_id = self.coordinator.reserve_experiment(ExperimentId(experiment, category), 30, 5, True, '{}', {}, {})
result = methods.get_experiment_status.call(category, experiment)
self.assertEquals({reservation_id : status}, result)
def test_list_sessions_not_existing(self):
exp_id = ExperimentId("exp.that.doesnt.exist","cat1")
self.assertRaises(
CoordExc.ExperimentNotFoundError,
self.reservations_manager.list_sessions,
exp_id
)
def test_reserve_experiment_repeated_uuid(self):
uuid = self.cfg_manager.get_doc_value(
configuration_doc.CORE_UNIVERSAL_IDENTIFIER)
status = self.processor.reserve_experiment(
ExperimentId('ud-dummy', 'Dummy experiments'), "{}",
'{ "%s" : [["%s","server x"]]}' %
(UserProcessor.SERVER_UUIDS, uuid), "127.0.0.1", uuid)
self.assertEquals('replicated', status)
def _initial_clean(self, coordination_configuration_parser):
external_servers_config = coordination_configuration_parser.parse_external_servers(
)
for external_server_str in external_servers_config:
for resource_type_name in external_servers_config[
external_server_str]:
self.resources_manager.add_experiment_id(
ExperimentId.parse(external_server_str),
resource_type_name)
def test_reserve_experiment_repeated_uuid(self):
uuid = self.cfg_manager.get_value(
WEBLAB_CORE_SERVER_UNIVERSAL_IDENTIFIER)
status = self.processor.reserve_experiment(
ExperimentId('ud-dummy', 'Dummy experiments'), "{}",
'{ "%s" : [["%s","server x"]]}' %
(UserProcessor.SERVER_UUIDS, uuid),
ClientAddress.ClientAddress("127.0.0.1"), uuid)
self.assertEquals('replicated', status)
def test_get_resource_types_by_experiment_id_error(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
self.assertRaises(
CoordExc.ExperimentNotFoundError,
self.resources_manager.get_resource_types_by_experiment_id,
ExperimentId("foo", "bar"))
def test_get_reservation_id_one_user(self):
category = "Dummy experiments"
experiment = "ud-dummy"
db_sess_id = DatabaseSession.ValidDatabaseSessionId('student2', "student")
sess_id, _ = self.ups.do_reserve_session(db_sess_id)
self.ups.reserve_experiment(sess_id, ExperimentId( experiment, category ), "{}", "{}", ClientAddress.ClientAddress( "127.0.0.1" ))
reservation_id = methods.get_reservation_id.call(sess_id.id)
self.assertNotEquals(None, reservation_id)
def _initial_clean(self, coordination_configuration_parser):
session = self._session_maker()
try:
external_servers_config = coordination_configuration_parser.parse_external_servers()
for external_server_str in external_servers_config:
for resource_type_name in external_servers_config[external_server_str]:
self.resources_manager.add_experiment_id(session, ExperimentId.parse(external_server_str), resource_type_name)
session.commit()
finally:
session.close()
def test_get_reservation_id_one_user(self):
category = "Dummy experiments"
experiment = "ud-dummy"
db_sess_id = ValidDatabaseSessionId('student2', "student")
sess_id, _ = self.ups._reserve_session(db_sess_id)
with wlcontext(self.ups, session_id = sess_id):
core_api.reserve_experiment(ExperimentId( experiment, category ), "{}", "{}")
reservation_id = methods.get_reservation_id.call(sess_id.id)
self.assertNotEquals(None, reservation_id)
def test_get_resource_types_by_experiment_id(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
exp_type_id = ExperimentId("ud-pld", "PLD Experiments")
resource_types = self.resources_manager.get_resource_types_by_experiment_id(
exp_type_id)
self.assertEquals(1, len(resource_types))
self.assertTrue(u"type" in resource_types)
def setUp(self):
cfg_manager = ConfigurationManager.ConfigurationManager()
cfg_manager.append_module(configuration)
self.dbmanager = DatabaseGateway(cfg_manager)
self.dbmanager._delete_all_uses()
session = self.dbmanager.Session()
try:
student1 = self.dbmanager._get_user(session, 'student1')
finally:
session.close()
self.initial_store = TemporalInformationStore.InitialTemporalInformationStore(
)
self.finished_store = TemporalInformationStore.FinishTemporalInformationStore(
)
self.commands_store = TemporalInformationStore.CommandsTemporalInformationStore(
)
self.completed_store = TemporalInformationStore.CompletedInformationStore(
)
self.retriever = TemporalInformationRetriever.TemporalInformationRetriever(
cfg_manager, self.initial_store, self.finished_store,
self.commands_store, self.completed_store, self.dbmanager)
self.retriever.timeout = 0.001 # Be quicker instead of waiting for half a second
self.initial_time = self.end_time = datetime.datetime.now()
self.initial_timestamp = self.end_timestamp = time.time()
request_info = {
'username': '******',
'role': 'student',
'permission_scope': 'user',
'permission_id': student1.id
}
exp_id = ExperimentId('ud-dummy', 'Dummy Experiments')
self.entry1 = TemporalInformationStore.InitialInformationEntry(
RESERVATION1, exp_id, coord_addr('ser:inst@mach'), DATA1,
self.initial_time, self.end_time, request_info.copy(),
DATA_REQUEST1)
self.entry2 = TemporalInformationStore.InitialInformationEntry(
RESERVATION2, exp_id, coord_addr('ser:inst@mach'), DATA2,
self.initial_time, self.end_time, request_info.copy(),
DATA_REQUEST2)
self.entry3 = TemporalInformationStore.InitialInformationEntry(
RESERVATION3, exp_id, coord_addr('ser:inst@mach'), DATA3,
self.initial_time, self.end_time, request_info.copy(),
DATA_REQUEST3)
self.entry4 = TemporalInformationStore.InitialInformationEntry(
RESERVATION4, exp_id, coord_addr('ser:inst@mach'), DATA4,
self.initial_time, self.end_time, request_info.copy(),
DATA_REQUEST3)
def create_reservation_processor(self, faking_response=False):
if faking_response:
self._fake_simple_lab_response()
status = self.user_processor.reserve_experiment(
ExperimentId('ud-dummy', 'Dummy experiments'), "{}", "{}",
ClientAddress.ClientAddress("127.0.0.1"), 'uuid')
self.reservation_processor = ReservationProcessor(
self.cfg_manager,
SessionId.SessionId(status.reservation_id.split(';')[0]), {
'session_polling':
(time.time(), ReservationProcessor.EXPIRATION_TIME_NOT_SET),
'latest_timestamp':
0,
'experiment_id':
ExperimentId('ud-dummy', 'Dummy experiments'),
'creator_session_id':
'',
'reservation_id':
SessionId.SessionId(status.reservation_id.split(';')[0]),
}, self.coordinator, self.locator, self.commands_store)
def _process(self):
redis_client = self.redis_maker()
pending_results = []
for reservation_id in redis_client.hkeys(self.external_weblabdeusto_pending):
pending_result_str = redis_client.hget(self.external_weblabdeusto_pending, reservation_id)
if pending_result_str is not None:
pending_result = json.loads(pending_result_str)
pending_result['reservation_id'] = reservation_id
pending_results.append(pending_result)
if len(pending_results) > 0:
try:
session_id, client = self.create_client_func(None)
except urllib2.URLError:
# Remote server is down, try later
return
remote_reservation_ids = [ SessionId(pending_result['remote_reservation_id']) for pending_result in pending_results ]
results = client.get_experiment_uses_by_id(session_id, remote_reservation_ids)
for pending_result, result in zip(pending_results, results):
if result.is_alive():
continue
username = pending_result['username']
try:
request_info = pickle.loads(pending_result['serialized_request_info'].encode('utf-8'))
except Exception as e:
log.log(ResultsRetriever, log.level.Critical, "Probably serialized_request_info was truncated in %s" % pending_result)
log.log_exc(ResultsRetriever, log.level.Error)
request_info = {'error' : 'could not be stored: %s' % e}
reservation_id = pending_result['reservation_id']
remote_reservation_id = pending_result['remote_reservation_id']
if not redis_client.hdel(self.external_weblabdeusto_pending, reservation_id):
log.log(ResultsRetriever, log.level.Info, "Pending reservation %r not found. Assuming it is managed by other thread" % pending_result)
continue
if result.is_finished():
use = result.experiment_use
use.experiment_id = ExperimentId.parse(pending_result['experiment_id_str'])
use.remote_reservation_id = remote_reservation_id
use.reservation_id = reservation_id
for key in [ key for key in request_info ]:
if not isinstance(request_info[key], (basestring, numbers.Number)):
request_info.pop(key)
use.request_info = request_info
callback = lambda : self.post_reservation_data_manager.delete(reservation_id)
self.completed_store.put(username, use, callback)
else:
log.log(ResultsRetriever, log.level.Info, "Reservation id %s was cancelled and therefore not stored" % reservation_id)
def process_GetLabConfiguration(self):
lab_server_id = serializer.parse_get_lab_configuration_request(
request.data)
ilab_request = {
'operation': 'get_lab_configuration',
}
# TODO: client address
reservation_status = weblab_api.api.reserve_experiment(
ExperimentId(lab_server_id, 'iLab experiments'),
json.dumps(ilab_request), '{}')
lab_configuration = reservation_status.initial_data
return serializer.generate_lab_configuration_response(lab_configuration)
def do_full_experiment_use(self, user_number):
"""
Uses the configured experiment trying to resemble the way a human would do it.
This method will block for a while.
:return:
"""
client = self.clients[user_number]
sessionid = client.login(self.user, self.password)
if not sessionid:
raise Exception("Wrong login")
# Reserve the flash dummy experiment.
experiment_id = ExperimentId(self.exp_name, self.cat_name)
waiting = client.reserve_experiment(sessionid, experiment_id, "{}",
"{}", None)
# print "Reserve response: %r" % waiting
reservation_id = waiting.reservation_id
initial_time = time.time()
while (time.time() - initial_time) < self.max_time:
status = client.get_reservation_status(reservation_id)
if type(status) is WaitingReservation:
time.sleep(0.1)
elif type(status) is ConfirmedReservation:
break
elif type(status) is WaitingConfirmationReservation:
time.sleep(0.1)
if (time.time() - initial_time) >= self.max_time:
raise Exception(
"Max time (%s seconds) achieved and still waiting..." %
self.max_time)
self.users_in += 1
# Send some commands.
for i in range(20):
# What's commandstring actually for??
cmd = Command("foo")
result = client.send_command(reservation_id, cmd)
if not result.commandstring.startswith("Received command"):
raise Exception("Unrecognized command response")
# print "Command result: %r" % result
time.sleep(0.1)
self.users_in -= 1
result = client.logout(sessionid)
def reserve_experiment(self, reservation_id, experiment_id, time, priority, initialization_in_accounting, client_initial_data, request_info):
server_uuids = list(request_info.get(SERVER_UUIDS, []))
server_uuids.append((self.core_server_uuid, self.core_server_uuid_human))
consumer_data = {
'time_allowed' : time,
'priority' : priority,
'initialization_in_accounting' : initialization_in_accounting,
'external_user' : request_info.get('username', ''),
SERVER_UUIDS : server_uuids,
}
for forwarded_key in FORWARDED_KEYS:
if forwarded_key in request_info:
consumer_data[forwarded_key] = request_info[forwarded_key]
consumer_data['external_user_unique'] = request_info.get('username_unique', request_info.get('username', ''))
# TODO: identifier of the server
login_client = self._create_login_client()
session_id = login_client.login(self.username, self.password)
client = self._create_client(login_client.get_cookies())
serialized_client_initial_data = json.dumps(client_initial_data)
serialized_consumer_data = json.dumps(consumer_data)
# If the administrator has mapped that this experiment_id is other, take that other. Otherwide, take the same one
requested_experiment_id_str = self.experiments_map.get(experiment_id.to_weblab_str(), experiment_id.to_weblab_str())
requested_experiment_id = ExperimentId.parse(requested_experiment_id_str)
external_reservation = client.reserve_experiment(session_id, requested_experiment_id, serialized_client_initial_data, serialized_consumer_data)
if external_reservation.is_null():
return None
remote_reservation_id = external_reservation.reservation_id.id
log.log(ExternalWebLabDeustoScheduler, log.level.Info, "Local reservation_id %s is linked to remote reservation %s" % (reservation_id, remote_reservation_id))
cookies = client.get_cookies()
serialized_cookies = pickle.dumps(cookies)
session = self.session_maker()
try:
reservation = ExternalWebLabDeustoReservation(reservation_id, remote_reservation_id, serialized_cookies, time_mod.time())
pending_results = ExternalWebLabDeustoReservationPendingResults(reservation_id, remote_reservation_id, self.resource_type_name, self.core_server_route, request_info.get('username', ''), pickle.dumps(request_info), experiment_id.to_weblab_str())
session.add(reservation)
session.add(pending_results)
session.commit()
finally:
session.close()
reservation_status = self._convert_reservation_to_status(external_reservation, reservation_id, remote_reservation_id)
return reservation_status
def process_GetLabConfiguration(self):
lab_server_id = self.serializer.parse_get_lab_configuration_request(
self.payload)
request = {
'operation': 'get_lab_configuration',
}
# TODO: client address
reservation_status = self.server.reserve_experiment(
self.sess_id, ExperimentId(lab_server_id, 'iLab experiments'),
json.dumps(request), '{}', ClientAddress.ClientAddress(''))
lab_configuration = reservation_status.initial_data
return self.serializer.generate_lab_configuration_response(
lab_configuration)
def test_list_experiments(self):
exp_id1 = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
resource_instance1 = Resource("type1", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id1, resource_instance1)
exp_id2 = ExperimentInstanceId("exp2", "ud-pld", "PLD Experiments")
resource_instance2 = Resource("type2", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id2, resource_instance2)
resources = self.resources_manager.list_experiments()
self.assertEquals(1, len(resources))
self.assertTrue(ExperimentId('ud-pld', 'PLD Experiments') in resources)
def reserve_experiment(experiment_id=None,
client_initial_data=None,
consumer_data=None):
server = weblab_api.ctx.server_instance
client_address = weblab_api.ctx.client_address
# core_server_universal_id should be copied
experiment_id = ExperimentId(experiment_id['exp_name'],
experiment_id['cat_name'])
status = weblab_api.ctx.user_processor.reserve_experiment(
experiment_id, client_initial_data, consumer_data, client_address,
server.core_server_universal_id)
if status == 'replicated':
return Reservation.NullReservation()
reservation_id = status.reservation_id.split(';')[0]
reservation_session_id = SessionId(reservation_id)
server._alive_users_collection.add_user(reservation_session_id)
session_id = server._reservations_session_manager.create_session(
reservation_id)
initial_session = {
'session_polling':
(time.time(), ReservationProcessor.EXPIRATION_TIME_NOT_SET),
'latest_timestamp':
0,
'experiment_id':
experiment_id,
'creator_session_id':
weblab_api.ctx.
user_session['session_id'], # Useful for monitor; should not be used
'reservation_id':
reservation_session_id,
'federated':
False,
}
reservation_processor = server._load_reservation(initial_session)
reservation_processor.update_latest_timestamp()
if status.status == WebLabSchedulingStatus.WebLabSchedulingStatus.RESERVED_LOCAL:
reservation_processor.process_reserved_status(status)
if status.status == WebLabSchedulingStatus.WebLabSchedulingStatus.RESERVED_REMOTE:
reservation_processor.process_reserved_remote_status(status)
server._reservations_session_manager.modify_session(
session_id, initial_session)
return Reservation.Reservation.translate_reservation(status)
def test_reject_experiment_voodoo_gen_raises_exception(self):
self.mock_locator[coord_addr(self.lab_address)]
self.mocker.throw(Exception("Unhandled exception"))
self.mocker.replay()
status, reservation1_id = self.coordinator.reserve_experiment(
ExperimentId('exp1', 'cat1'), 30, 5, True, 'sample initial data',
DEFAULT_REQUEST_INFO, {})
self.coordinator.confirmer._confirm_handler.join()
self.assertEquals(None, self.confirmer._confirm_handler.raised_exc)
status = self.coordinator.get_reservation_status(reservation1_id)
expected_status = WSS.WaitingInstancesQueueStatus(reservation1_id, 0)
self.assertEquals(expected_status, status)
def do_full_experiment_use():
"""
Uses the configured experiment trying to resemble the way a human would do it.
This method will block for a while.
:return:
"""
wc = WebLabDeustoClient(config.WEBLAB_BASE_URL)
sessionid = wc.login(config.LOGIN, config.PASSWORD)
if not sessionid: raise Exception("Wrong login")
# Reserve the flash dummy experiment.
experiment_id = ExperimentId(config.EXP_NAME, config.EXP_CATEGORY)
waiting = wc.reserve_experiment(sessionid, experiment_id, "{}", "{}", None)
# print "Reserve response: %r" % waiting
reservation_id = waiting.reservation_id
while True:
status = wc.get_reservation_status(reservation_id)
# print "Reservation status: %r" % status
if type(status) is WaitingReservation:
time.sleep(0.5)
elif type(status) is ConfirmedReservation:
break
elif type(status) is WaitingConfirmationReservation:
time.sleep(0.5)
else:
print "Unknown reservation status."
print "Experiment reserved."
global users_in
users_in += 1
# Send some commands.
for i in range(config.COMMANDS_PER_USER):
# What's commandstring actually for??
cmd = Command(config.COMMAND)
result = wc.send_command(reservation_id, cmd)
if not result.commandstring.startswith("Received command"):
raise Exception("Unrecognized command response")
# print "Command result: %r" % result
time.sleep(config.TIME_BETWEEN_COMMANDS)
users_in -= 1
result = wc.logout(sessionid)
print "Logout result: %r" % result
def _initial_clean(self, coordination_configuration_parser):
session = self._session_maker()
try:
external_servers_config = coordination_configuration_parser.parse_external_servers(
)
for external_server_str in external_servers_config:
for resource_type_name in external_servers_config[
external_server_str]:
self.resources_manager.add_experiment_id(
session, ExperimentId.parse(external_server_str),
resource_type_name)
session.commit()
finally:
session.close()
def direct2experiment():
experiment_id_str = get_argument(EXPERIMENT_ID)
if experiment_id_str is None:
return "%s argument is missing" % EXPERIMENT_ID
session_id_str = get_argument(SESSION_ID)
if session_id_str is None:
return "%s argument is missing" % EXPERIMENT_ID
experiment_id = ExperimentId.parse(experiment_id_str)
weblab_api.context.session_id = session_id_str
try:
reservation_id = weblab_api.api.reserve_experiment(experiment_id, "{}", "{}")
except Exception:
traceback.print_exc()
return HTML_ERROR_TEMPLATE
new_location = "../../client/federated.html#reservation_id=%s" % reservation_id.reservation_id.id
return redirect(new_location)
def run(self):
experiment_id_str = self.get_argument(EXPERIMENT_ID)
if experiment_id_str is None:
return "%s argument is missing" % EXPERIMENT_ID
session_id_str = self.get_argument(SESSION_ID)
if session_id_str is None:
return "%s argument is missing" % EXPERIMENT_ID
experiment_id = ExperimentId.parse(experiment_id_str)
session_id = SessionId(session_id_str)
address = RemoteFacadeContext.get_context().get_ip_address()
client_address = ClientAddress.ClientAddress(address)
try:
reservation_id = self.server.reserve_experiment(session_id, experiment_id, "{}", "{}", client_address)
except Exception:
return HTML_ERROR_TEMPLATE
new_location = "../../client/federated.html#reservation_id=%s" % reservation_id.reservation_id.id
self.set_status(302)
self.add_other_header('Location', new_location)
return """<html><body><a href="%s">Click here</a></body></html>""" % new_location
def list_laboratories_addresses(self):
client = self._redis_maker()
laboratory_addresses = {
# laboratory_coord_address : {
# experiment_instance_id : resource_instance
# }
}
for experiment_type in client.smembers(WEBLAB_EXPERIMENT_TYPES):
experiment_id = ExperimentId.parse(experiment_type)
experiment_instance_names = client.smembers(WEBLAB_EXPERIMENT_INSTANCES % experiment_type)
for experiment_instance_name in experiment_instance_names:
experiment_instance_id = ExperimentInstanceId(experiment_instance_name, experiment_id.exp_name, experiment_id.cat_name)
weblab_experiment_instance = WEBLAB_EXPERIMENT_INSTANCE % (experiment_type, experiment_instance_name)
laboratory_address = client.hget(weblab_experiment_instance, LAB_COORD)
resource_str = client.hget(weblab_experiment_instance, RESOURCE_INST)
resource = Resource.parse(resource_str)
current = laboratory_addresses.get(laboratory_address, {})
current[experiment_instance_id] = resource
laboratory_addresses[laboratory_address] = current
return laboratory_addresses
def __init__(self, data_manager, locator, cfg_manager, ConfirmerClass):
if ConfirmerClass is None:
ConfirmerClass = Confirmer.ReservationConfirmer
self.cfg_manager = cfg_manager
self._data_manager = data_manager
self.core_server_url = self.cfg_manager.get_value(CORE_SERVER_URL)
self.notifier = AdminNotifier.AdminNotifier(self.cfg_manager)
self.notifications_enabled = self.cfg_manager.get_value(RESOURCES_CHECKER_NOTIFICATIONS_ENABLED, DEFAULT_RESOURCES_CHECKER_NOTIFICATIONS_ENABLED)
self.locator = locator # Used by ResourcesChecker
self.confirmer = ConfirmerClass(self, locator)
self.time_provider = self.CoordinatorTimeProvider()
self.initial_store = TemporalInformationStore.InitialTemporalInformationStore()
self.finished_store = TemporalInformationStore.FinishTemporalInformationStore()
self.completed_store = TemporalInformationStore.CompletedInformationStore()
self.finished_reservations_store = Queue.Queue()
self._initialize_managers()
#
# The system administrator must define what scheduling system is used by each resource type
# For instance:
#
# scheduling_systems = {
# "pld boards" : ("PRIORITY_QUEUE", {}),
# "fpga boards" : ("PRIORITY_QUEUE", {}),
# "vm experiments" : ("BOOKING", { 'slots' : 30 * 1000 }), # Slots of 30 minutes
# "something else" : ("EXTERNAL", { 'address' : 'http://192.168.1.50:8080/SchedulingServer', 'protocol' : 'SOAP' }) # If somebody else has implemented the scheduling schema in other language
# }
#
self.schedulers = {}
scheduling_systems = cfg_manager.get_value(CORE_SCHEDULING_SYSTEMS)
for resource_type_name in scheduling_systems:
scheduling_system, arguments = scheduling_systems[resource_type_name]
if not scheduling_system in self.SCHEDULING_SYSTEMS:
raise CoordExc.UnregisteredSchedulingSystemError("Unregistered scheduling system: %r" % scheduling_system)
SchedulingSystemClass = self.SCHEDULING_SYSTEMS[scheduling_system]
data_manager = self._data_manager
generic_scheduler_arguments = Scheduler.GenericSchedulerArguments(
cfg_manager = self.cfg_manager,
resource_type_name = resource_type_name,
reservations_manager = self.reservations_manager,
resources_manager = self.resources_manager,
confirmer = self.confirmer,
data_manager = data_manager,
time_provider = self.time_provider,
core_server_url = self.core_server_url,
initial_store = self.initial_store,
finished_store = self.finished_store,
completed_store = self.completed_store,
post_reservation_data_manager = self.post_reservation_data_manager
)
self.schedulers[resource_type_name] = SchedulingSystemClass(generic_scheduler_arguments, **arguments)
self.aggregators = {
# experiment_id_str : AGGREGATOR( schedulers )
}
coordination_configuration_parser = CoordinationConfigurationParser.CoordinationConfigurationParser(cfg_manager)
resource_types_per_experiment_id = coordination_configuration_parser.parse_resources_for_experiment_ids()
#
# This configuration argument has a dictionary such as:
# {
# 'experiment_id_str' : {'foo' : 'bar'}
# }
#
# The argument itself is not mandatory.
#
aggregators_configuration = self.cfg_manager.get_value(CORE_SCHEDULER_AGGREGATORS, {})
for experiment_id_str in resource_types_per_experiment_id:
generic_scheduler_arguments = Scheduler.GenericSchedulerArguments(
cfg_manager = self.cfg_manager,
resource_type_name = None,
reservations_manager = self.reservations_manager,
resources_manager = self.resources_manager,
confirmer = self.confirmer,
data_manager = self._data_manager,
time_provider = self.time_provider,
core_server_url = self.core_server_url,
initial_store = self.initial_store,
finished_store = self.finished_store,
completed_store = self.completed_store,
post_reservation_data_manager = self.post_reservation_data_manager
)
resource_type_names = resource_types_per_experiment_id[experiment_id_str]
try:
aggregated_schedulers = {}
for resource_type_name in resource_type_names:
aggregated_schedulers[resource_type_name] = self.schedulers[resource_type_name]
except KeyError, ke:
raise Exception("Scheduler not found with resource type name %s. Check %s config property." % (ke, CORE_SCHEDULING_SYSTEMS))
particular_configuration = aggregators_configuration.get(experiment_id_str)
aggregator = self.AGGREGATOR(generic_scheduler_arguments, ExperimentId.parse(experiment_id_str), aggregated_schedulers, particular_configuration)
self.aggregators[experiment_id_str] = aggregator
def get_experiment_id(self, reservation_id):
reservation_data = self.get_reservation_data(reservation_id)
if reservation_data is None:
raise CoordExc.ExpiredSessionError("Expired reservation: no experiment id found for that reservation (%s)" % reservation_id)
return ExperimentId.parse(reservation_data[EXPERIMENT_TYPE])
def _process(self):
session = self.session_maker()
try:
pending_results = [ pending_result.to_dto() for pending_result in session.query(ExternalWebLabDeustoReservationPendingResults).filter_by(resource_type_name = self.resource_type_name, server_route = self.server_route).all() ]
finally:
session.close()
if len(pending_results) > 0:
try:
session_id, client = self.create_client_func(None)
except urllib2.URLError:
# Remote server is down, try later
return
remote_reservation_ids = [ SessionId(pending_result.remote_reservation_id) for pending_result in pending_results ]
results = client.get_experiment_uses_by_id(session_id, remote_reservation_ids)
for pending_result, result in zip(pending_results, results):
if result.is_alive():
continue
username = pending_result.username
try:
request_info = pickle.loads(pending_result.serialized_request_info.encode('utf-8'))
except Exception as e:
log.log(ResultsRetriever, log.level.Critical, "Probably serialized_request_info was truncated in %s" % pending_result)
log.log_exc(ResultsRetriever, log.level.Error)
request_info = {'error' : 'could not be stored: %s' % e}
reservation_id = pending_result.reservation_id
remote_reservation_id = pending_result.remote_reservation_id
session = self.session_maker()
try:
db_pending_result = session.query(ExternalWebLabDeustoReservationPendingResults).filter_by(id = pending_result.id).first()
if db_pending_result is not None:
session.delete(db_pending_result)
session.commit()
else:
log.log(ResultsRetriever, log.level.Info, "Pending reservation %r not found. Assuming it is managed by other thread" % pending_result)
continue
except (IntegrityError, ConcurrentModificationError, StaleDataError):
log.log(ResultsRetriever, log.level.Info, "Pending reservation %r deletion failed. Assuming it is managed by other thread" % pending_result)
log.log_exc(ResultsRetriever, log.level.Debug)
# Somebody else is already handling this
continue
finally:
session.close()
if result.is_finished():
use = result.experiment_use
use.experiment_id = ExperimentId.parse(pending_result.experiment_id_str)
use.remote_reservation_id = remote_reservation_id
use.reservation_id = reservation_id
for key in [ key for key in request_info ]:
if not isinstance(request_info[key], (basestring, numbers.Number)):
request_info.pop(key)
use.request_info = request_info
callback = lambda : self.post_reservation_data_manager.delete(reservation_id)
self.completed_store.put(username, use, callback)
else:
log.log(ResultsRetriever, log.level.Info, "Reservation id %s was cancelled and therefore not stored" % reservation_id)
def _initial_clean(self, coordination_configuration_parser):
external_servers_config = coordination_configuration_parser.parse_external_servers()
for external_server_str in external_servers_config:
for resource_type_name in external_servers_config[external_server_str]:
self.resources_manager.add_experiment_id(ExperimentId.parse(external_server_str), resource_type_name)
def list_experiments(self):
client = self._redis_maker()
return [ ExperimentId.parse(exp_type) for exp_type in client.smembers(WEBLAB_EXPERIMENT_TYPES) ]
def _update_queues(self):
###########################################################
# There are reasons why a waiting reservation may not be
# able to be promoted while the next one is. For instance,
# if a user is waiting for "pld boards", but only for
# instances of "pld boards" which have a "ud-binary@Binary
# experiments" server running. If only a "ud-pld@PLD
# Experiments" is available, then this user will not be
# promoted and the another user which is waiting for a
# "ud-pld@PLD Experiments" can be promoted.
#
# Therefore, we have a list of the IDs of the waiting
# reservations we previously thought that they couldn't be
# promoted in this iteration. They will have another
# chance in the next run of _update_queues.
#
previously_waiting_reservation_ids = []
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
weblab_resource_slots = WEBLAB_RESOURCE_SLOTS % self.resource_type_name
###########################################################
# While there are free instances and waiting reservations,
# take the first waiting reservation and set it to current
# reservation. Make this repeatedly because we want to
# commit each change
#
while True:
client = self.redis_maker()
filled_waiting_reservation_ids = client.zrangebyscore(weblab_resource_pqueue_sorted, -10000, +10000, start=0, num=len(previously_waiting_reservation_ids) + 1)
first_waiting_reservation_id = None
for filled_waiting_reservation_id in filled_waiting_reservation_ids:
waiting_reservation_id = filled_waiting_reservation_id[filled_waiting_reservation_id.find('_')+1:]
if waiting_reservation_id not in previously_waiting_reservation_ids:
first_waiting_reservation_id = waiting_reservation_id
break
if first_waiting_reservation_id is None:
return # There is no waiting reservation for this resource that we haven't already tried
previously_waiting_reservation_ids.append(first_waiting_reservation_id)
#
# For the current resource_type, let's ask for
# all the resource instances available (i.e. those
# who are a member on weblab:resource:%s:slots )
#
free_instances = [ Resource(self.resource_type_name, resource_instance)
for resource_instance in client.smembers(weblab_resource_slots) ]
if len(free_instances) == 0:
# If there is no free instance, just return
return
#
# Select the correct free_instance for the current student among
# all the free_instances
#
if self.randomize_instances:
randomized_free_instances = [ free_instance for free_instance in free_instances ]
random.shuffle(randomized_free_instances)
else:
randomized_free_instances = sorted(free_instances, cmp=lambda r1, r2: cmp(r1.resource_type, r2.resource_type) or cmp(r1.resource_instance, r2.resource_instance))
for free_instance in randomized_free_instances:
#
# IMPORTANT: from here on every "continue" should first revoke the
# reservations_manager and resources_manager confirmations
#
working = self.resources_manager.check_working(free_instance)
if not working:
# The instance is not working
continue
confirmed = self.reservations_manager.confirm(first_waiting_reservation_id)
if not confirmed:
# student has already been confirmed somewhere else, so don't try with other
# instances, but rather with other student
break
acquired = self.resources_manager.acquire_resource(free_instance)
# print "ACQUIRED", free_instance, acquired, time.time()
if not acquired:
# the instance has been acquired by someone else. unconfirm student and
# try again with other free_instance
self.reservations_manager.downgrade_confirmation(first_waiting_reservation_id)
continue
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (self.resource_type_name, free_instance.resource_instance)
client.sadd(weblab_resource_pqueue_instance_reservations, first_waiting_reservation_id)
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, first_waiting_reservation_id)
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
reservation_data = self.reservations_manager.get_reservation_data(first_waiting_reservation_id)
if pqueue_reservation_data_str is None or reservation_data is None:
# the student is not here anymore; downgrading confirmation is not required
# but releasing the resource is; and skip the rest of the free instances
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations, first_waiting_reservation_id)
break
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
start_time = self.time_provider.get_time()
total_time = pqueue_reservation_data[TIME]
pqueue_reservation_data[START_TIME] = start_time
pqueue_reservation_data[TIMESTAMP_BEFORE] = start_time
pqueue_reservation_data[ACTIVE_STATUS] = STATUS_WAITING_CONFIRMATION
pqueue_reservation_data[RESOURCE_INSTANCE] = free_instance.to_weblab_str()
initialization_in_accounting = pqueue_reservation_data[INITIALIZATION_IN_ACCOUNTING]
client_initial_data = reservation_data[CLIENT_INITIAL_DATA]
request_info = json.loads(reservation_data[REQUEST_INFO])
username = request_info.get('username')
locale = request_info.get('locale')
requested_experiment_type = ExperimentId.parse(reservation_data[EXPERIMENT_TYPE])
selected_experiment_instance = None
experiment_instances = self.resources_manager.list_experiment_instance_ids_by_resource(free_instance)
for experiment_instance in experiment_instances:
if experiment_instance.to_experiment_id() == requested_experiment_type:
selected_experiment_instance = experiment_instance
if selected_experiment_instance is None:
# This resource is not valid for this user, other free_instance should be
# selected. Try with other, but first clean the acquired resources
self.reservations_manager.downgrade_confirmation(first_waiting_reservation_id)
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations, first_waiting_reservation_id)
continue
pqueue_reservation_data[EXPERIMENT_INSTANCE] = selected_experiment_instance.to_weblab_str()
laboratory_coord_address = self.resources_manager.get_laboratory_coordaddress_by_experiment_instance_id(selected_experiment_instance)
pqueue_reservation_data[LAB_COORD] = laboratory_coord_address
client.set(weblab_reservation_pqueue, json.dumps(pqueue_reservation_data))
filled_reservation_id = client.hget(weblab_resource_pqueue_map, first_waiting_reservation_id)
client.zrem(weblab_resource_pqueue_sorted, filled_reservation_id)
#
# Enqueue the confirmation, since it might take a long time
# (for instance, if the laboratory server does not reply because
# of any network problem, or it just takes too much in replying),
# so this method might take too long. That's why we enqueue these
# petitions and run them in other threads.
#
deserialized_server_initial_data = {
'priority.queue.slot.length' : '%s' % total_time,
'priority.queue.slot.start' : '%s' % datetime.datetime.fromtimestamp(start_time),
'priority.queue.slot.initialization_in_accounting' : initialization_in_accounting,
'request.experiment_id.experiment_name' : selected_experiment_instance.exp_name,
'request.experiment_id.category_name' : selected_experiment_instance.cat_name,
'request.username' : username,
'request.full_name' : username,
'request.locale' : locale,
# TODO: add the username and user full name here
}
server_initial_data = json.dumps(deserialized_server_initial_data)
# server_initial_data will contain information such as "what was the last experiment used?".
# If a single resource was used by a binary experiment, then the next time may not require reprogramming the device
self.confirmer.enqueue_confirmation(laboratory_coord_address, first_waiting_reservation_id, selected_experiment_instance, client_initial_data, server_initial_data, self.resource_type_name)
#
# After it, keep in the while True in order to add the next
# reservation
#
break