def test_add_resource(self):
session = self.session_maker()
try:
resource_types = session.query(CoordinatorModel.ResourceType).all()
self.assertEquals(
0, len(resource_types),
"No resource expected in the beginning of the test")
self.resources_manager.add_resource(session,
Resource("type", "instance"))
self._check_resource_added(session)
session.commit()
finally:
session.close()
# Can be executed twice without conflicts
session = self.session_maker()
try:
self.resources_manager.add_resource(session,
Resource("type", "instance"))
self._check_resource_added(session)
session.commit()
finally:
session.close()
def test_list_experiment_instance_ids_by_resource(self):
session = self.session_maker()
try:
exp_id1 = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
resource_instance1 = Resource("type1", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id1,
resource_instance1)
exp_id2 = ExperimentInstanceId("exp2", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id2,
resource_instance1)
exp_id3 = ExperimentInstanceId("exp3", "ud-pld", "PLD Experiments")
resource_instance2 = Resource("type1", "instance2")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id3,
resource_instance2)
session.commit()
finally:
session.close()
experiment_instance_ids = self.resources_manager.list_experiment_instance_ids_by_resource(
resource_instance1)
self.assertEquals(2, len(experiment_instance_ids))
self.assertTrue(
ExperimentInstanceId('exp1', 'ud-pld', 'PLD Experiments') in
experiment_instance_ids)
self.assertTrue(
ExperimentInstanceId('exp2', 'ud-pld', 'PLD Experiments') in
experiment_instance_ids)
def test_add_experiment_instance_id_redundant(self):
self.assertEquals([], self.resources_manager.list_resources())
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
# No problem in adding twice the same
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
# Everything is all right
self._check_resource_added()
self._check_experiment_instance_id_added()
# However, we can't add another time the same experiment instance with a different laboratory id:
self.assertRaises(CoordExc.InvalidExperimentConfigError,
self.resources_manager.add_experiment_instance_id,
"laboratory2:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
# Or the same experiment instance with a different resource instance:
self.assertRaises(CoordExc.InvalidExperimentConfigError,
self.resources_manager.add_experiment_instance_id,
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance2"))
def test_add_resource(self):
self.assertEquals([], self.resources_manager.list_resources())
self.resources_manager.add_resource(Resource("type", "instance"))
self._check_resource_added()
# Can be executed twice without conflicts
self.resources_manager.add_resource(Resource("type", "instance"))
self._check_resource_added()
def test_get_resource_instance_by_experiment_instance_id(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
resource = self.resources_manager.get_resource_instance_by_experiment_instance_id(
exp_id)
expected_resource = Resource("type", "instance")
self.assertEquals(expected_resource, resource)
def test_eq_other(self):
res1 = Resource("foo", "bar")
res2 = Resource("foo", "bar2")
self.assertNotEquals(res1, res2)
res2 = Resource("foo2", "bar")
self.assertNotEquals(res1, res2)
res2 = "foobar"
self.assertNotEquals(res1, res2)
def test_list_laboratories_addresses(self):
session = self.session_maker()
try:
exp_id1 = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
resource_instance1 = Resource("type1", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id1,
resource_instance1)
# Repeating laboratory1, but a set is returned so no problem
exp_id2 = ExperimentInstanceId("exp2", "ud-pld", "PLD Experiments")
resource_instance2 = Resource("type2", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id2,
resource_instance2)
exp_id3 = ExperimentInstanceId("exp3", "ud-pld", "PLD Experiments")
resource_instance3 = Resource("type2", "instance2")
self.resources_manager.add_experiment_instance_id(
"laboratory2:WL_SERVER1@WL_MACHINE1", exp_id3,
resource_instance3)
session.commit()
finally:
session.close()
addresses = self.resources_manager.list_laboratories_addresses()
self.assertEquals(2, len(addresses))
self.assertTrue("laboratory1:WL_SERVER1@WL_MACHINE1" in addresses)
self.assertEquals(2,
len(addresses["laboratory1:WL_SERVER1@WL_MACHINE1"]))
self.assertTrue(
exp_id1 in addresses["laboratory1:WL_SERVER1@WL_MACHINE1"])
self.assertTrue(
exp_id2 in addresses["laboratory1:WL_SERVER1@WL_MACHINE1"])
self.assertTrue("laboratory2:WL_SERVER1@WL_MACHINE1" in addresses)
self.assertEquals(1,
len(addresses["laboratory2:WL_SERVER1@WL_MACHINE1"]))
self.assertTrue(
exp_id3 in addresses["laboratory2:WL_SERVER1@WL_MACHINE1"])
self.assertEquals(
resource_instance1,
addresses["laboratory1:WL_SERVER1@WL_MACHINE1"][exp_id1])
self.assertEquals(
resource_instance2,
addresses["laboratory1:WL_SERVER1@WL_MACHINE1"][exp_id2])
self.assertEquals(
resource_instance3,
addresses["laboratory2:WL_SERVER1@WL_MACHINE1"][exp_id3])
def test_list_experiments(self):
exp_id1 = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
resource_instance1 = Resource("type1", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id1, resource_instance1)
exp_id2 = ExperimentInstanceId("exp2", "ud-pld", "PLD Experiments")
resource_instance2 = Resource("type2", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id2, resource_instance2)
resources = self.resources_manager.list_experiments()
self.assertEquals(1, len(resources))
self.assertTrue(ExperimentId('ud-pld', 'PLD Experiments') in resources)
def test_get_resource_instance_by_experiment_instance_id(self):
session = self.session_maker()
try:
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
session.commit()
finally:
session.close()
resource = self.resources_manager.get_resource_instance_by_experiment_instance_id(
exp_id)
expected_resource = Resource("type", "instance")
self.assertEquals(expected_resource, resource)
def list_resource_instances_by_type(self, resource_type_name):
client = self._redis_maker()
return [
Resource(resource_type_name, resource_instance)
for resource_instance in client.smembers(WEBLAB_RESOURCE %
resource_type_name)
]
def test_remove_resource_instance(self):
session = self.session_maker()
try:
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
resource_instance = Resource("type", "instance")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
resource_instance)
# Checking that the resources are there
experiment_instances = session.query(
CoordinatorModel.ExperimentInstance).all()
self.assertEquals(1, len(experiment_instances))
resource_instances = session.query(
CoordinatorModel.ResourceInstance).all()
self.assertEquals(1, len(resource_instances))
# Removing resource instance
self.resources_manager.remove_resource_instance(
session, resource_instance)
# Checking that the resources are not there, neither the experiment instances
resource_instances = session.query(
CoordinatorModel.ResourceInstance).all()
self.assertEquals(0, len(resource_instances))
experiment_instances = session.query(
CoordinatorModel.ExperimentInstance).all()
self.assertEquals(0, len(experiment_instances))
session.commit()
finally:
session.close()
def setUp(self):
self.mocker = mocker.Mocker()
self.lab_mock = self.mocker.mock()
self.locator = FakeLocator(lab=self.lab_mock)
self.db = FakeDatabase()
self.cfg_manager = ConfigurationManager.ConfigurationManager()
self.cfg_manager.append_module(configuration_module)
self.cfg_manager._set_value(
COORDINATOR_LABORATORY_SERVERS, {
'server:laboratoryserver@labmachine': {
'inst1|ud-dummy|Dummy experiments': 'res_inst@res_type'
}
})
self.commands_store = TemporalInformationStore.CommandsTemporalInformationStore(
)
self.coordinator = coordinator_create(SQLALCHEMY,
self.locator,
self.cfg_manager,
ConfirmerClass=FakeConfirmer)
self.coordinator._clean()
self.coordinator.add_experiment_instance_id(
"server:laboratoryserver@labmachine",
ExperimentInstanceId('inst', 'ud-dummy', 'Dummy experiments'),
Resource("res_type", "res_inst"))
self.processor = UserProcessor.UserProcessor(
self.locator,
{'db_session_id': ValidDatabaseSessionId('my_db_session_id')},
self.cfg_manager, self.coordinator, self.db, self.commands_store)
def list_resource_instances(self):
client = self._redis_maker()
resource_instances = []
for resource_type in client.smembers(WEBLAB_RESOURCES):
for resource_instance in client.smembers(WEBLAB_RESOURCE % resource_type):
resource_instances.append(Resource(resource_type, resource_instance))
return resource_instances
def test_get_resource_types_by_experiment_id_error(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
self.assertRaises(
CoordExc.ExperimentNotFoundError,
self.resources_manager.get_resource_types_by_experiment_id,
ExperimentId("foo", "bar"))
def test_add_experiment_instance_id_redundant(self):
session = self.session_maker()
try:
resource_types = session.query(CoordinatorModel.ResourceType).all()
self.assertEquals(
0, len(resource_types),
"No resource expected in the beginning of the test")
finally:
session.close()
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
# No problem in adding twice the same
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
session = self.session_maker()
try:
# Everything is all right
self._check_resource_added(session)
self._check_experiment_instance_id_added(session)
# However, we can't add another time the same experiment instance with a different laboratory id:
self.assertRaises(
CoordExc.InvalidExperimentConfigError,
self.resources_manager.add_experiment_instance_id,
"laboratory2:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
# Or the same experiment instance with a different resource instance:
self.assertRaises(
CoordExc.InvalidExperimentConfigError,
self.resources_manager.add_experiment_instance_id,
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance2"))
session.commit()
finally:
session.close()
def test_get_resource_types_by_experiment_id(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
exp_type_id = ExperimentId("ud-pld", "PLD Experiments")
resource_types = self.resources_manager.get_resource_types_by_experiment_id(
exp_type_id)
self.assertEquals(1, len(resource_types))
self.assertTrue(u"type" in resource_types)
def test_coordination_configuration_parser(self):
self.cfg_manager._set_value(CoordinationConfigurationParser.COORDINATOR_LABORATORY_SERVERS, {
'laboratory1:WL_SERVER1@WL_MACHINE1' : {
'exp1|ud-fpga|FPGA experiments' : 'fpga1@fpga boards',
'exp1|ud-pld|PLD experiments' : 'pld1@pld boards',
},
})
configuration = self.coordination_configuration_parser.parse_configuration()
self.assertEquals(1, len(configuration))
lab_config = configuration['laboratory1:WL_SERVER1@WL_MACHINE1']
self.assertEquals(2, len(lab_config))
exp_fpga = ExperimentInstanceId("exp1","ud-fpga","FPGA experiments")
exp_pld = ExperimentInstanceId("exp1","ud-pld","PLD experiments")
fpga_resource = lab_config[exp_fpga]
self.assertEquals(Resource("fpga boards", "fpga1"), fpga_resource)
pld_resource = lab_config[exp_pld]
self.assertEquals(Resource("pld boards", "pld1"), pld_resource)
def setUp(self):
locator_mock = None
self.cfg_manager = ConfigurationManager.ConfigurationManager()
self.cfg_manager.append_module(configuration_module)
self.coordinator = WrappedSqlCoordinator(locator_mock,
self.cfg_manager,
ConfirmerClass=ConfirmerMock)
self.coordinator._clean()
self.coordinator.add_experiment_instance_id(
"lab1:inst@machine", ExperimentInstanceId('inst1', 'exp1', 'cat1'),
Resource("res_type", "res_inst1"))
self.coordinator.add_experiment_instance_id(
"lab2:inst@machine", ExperimentInstanceId('inst2', 'exp2', 'cat1'),
Resource("res_type", "res_inst2"))
self.reservations_manager = self.coordinator.reservations_manager
def test_add_experiment_instance_id(self):
self.assertEquals([], self.resources_manager.list_resources())
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
self._check_resource_added()
self._check_experiment_instance_id_added()
def test_list_resources(self):
session = self.session_maker()
try:
exp_id1 = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
resource_instance1 = Resource("type1", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id1,
resource_instance1)
exp_id2 = ExperimentInstanceId("exp2", "ud-pld", "PLD Experiments")
resource_instance2 = Resource("type2", "instance1")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id2,
resource_instance2)
session.commit()
finally:
session.close()
resources = self.resources_manager.list_resources()
self.assertEquals(2, len(resources))
self.assertTrue('type1' in resources)
self.assertTrue('type2' in resources)
def parse_configuration(self):
#
# configuration = {
# "laboratory1:WL_SERVER1@WL_MACHINE1" : {
# ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments") : ("pld1", "ud-pld-boards")
# }
# }
#
configuration = {}
laboratory_servers = self._cfg_manager.get_value(
COORDINATOR_LABORATORY_SERVERS)
for laboratory_server_coord_address_str in laboratory_servers:
experiment_instances = laboratory_servers[
laboratory_server_coord_address_str]
laboratory_configuration = {}
configuration[
laboratory_server_coord_address_str] = laboratory_configuration
for experiment_instance in experiment_instances:
resource_instance = experiment_instances[experiment_instance]
mo_experiment_instance = re.match(
self.EXPERIMENT_INSTANCE_REGEX, experiment_instance)
if mo_experiment_instance is None:
raise coreExc.CoordinationConfigurationParsingError(
"Error in coordination parsing: %s doesn't match the regular expression %s"
%
(experiment_instance, self.EXPERIMENT_INSTANCE_REGEX))
mo_resource_instance = re.match(self.RESOURCE_INSTANCE_REGEX,
resource_instance)
if mo_resource_instance is None:
raise coreExc.CoordinationConfigurationParsingError(
"Error in coordination parsing: %s doesn't match the regular expression %s"
% (resource_instance, self.RESOURCE_INSTANCE_REGEX))
(inst_name, exp_name,
exp_cat_name) = mo_experiment_instance.groups()
experiment_instance_id = ExperimentInstanceId(
inst_name, exp_name, exp_cat_name)
(resource_instance,
resource_type) = mo_resource_instance.groups()
resource = Resource(resource_type, resource_instance)
laboratory_configuration[experiment_instance_id] = resource
return configuration
def test_remove_resource_instance_id(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
experiment_instances = self.resources_manager.list_experiment_instances_by_type(
exp_id.to_experiment_id())
self.assertEquals(1, len(experiment_instances))
self.resources_manager.remove_resource_instance_id(exp_id)
experiment_instances = self.resources_manager.list_experiment_instances_by_type(
exp_id.to_experiment_id())
self.assertEquals(0, len(experiment_instances))
def test_scheduler_reservation_associations(self):
exp_inst_id1 = ExperimentInstanceId("exp1", "ud-pld",
"PLD experiments")
exp_inst_id1b = ExperimentInstanceId("exp2", "ud-pld",
"PLD experiments")
exp_inst_id2 = ExperimentInstanceId("exp1", "ud-fpga",
"FPGA experiments")
exp_id1 = exp_inst_id1.to_experiment_id()
exp_id2 = exp_inst_id2.to_experiment_id()
self.resources_manager.add_resource(Resource("pld_local", "instance"))
self.resources_manager.add_resource(Resource("pld_remote", "instance"))
self.resources_manager.add_resource(Resource("fpga_remote",
"instance"))
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_inst_id1,
Resource("pld_local", "instance"))
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_inst_id1b,
Resource("pld_remote", "instance"))
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_inst_id2,
Resource("fpga_remote", "instance"))
reservation1 = 'reservation1'
reservation2 = 'reservation2'
self.resources_manager.associate_scheduler_to_reservation(
reservation1, exp_id1, 'pld_local')
self.resources_manager.associate_scheduler_to_reservation(
reservation1, exp_id1, 'pld_remote')
self.resources_manager.associate_scheduler_to_reservation(
reservation2, exp_id2, 'fpga_remote')
resource_type_names = self.resources_manager.retrieve_schedulers_per_reservation(
reservation1, exp_id1)
self.assertEquals(set(('pld_local', 'pld_remote')),
set(resource_type_names))
resource_type_names = self.resources_manager.retrieve_schedulers_per_reservation(
reservation2, exp_id2)
self.assertEquals(['fpga_remote'], list(resource_type_names))
self.resources_manager.dissociate_scheduler_from_reservation(
reservation1, exp_id1, 'pld_remote')
resource_type_names = self.resources_manager.retrieve_schedulers_per_reservation(
reservation1, exp_id1)
self.assertEquals(['pld_local'], list(resource_type_names))
self.resources_manager.clean_associations_for_reservation(
reservation1, exp_id1)
resource_type_names = self.resources_manager.retrieve_schedulers_per_reservation(
reservation1, exp_id1)
self.assertEquals(0, len(resource_type_names))
def test_get_resource_instance_by_experiment_instance_id_failing(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
exp_invalid_type = ExperimentInstanceId("exp1", "ud-pld.invalid",
"PLD Experiments")
self.assertRaises(
CoordExc.ExperimentNotFoundError, self.resources_manager.
get_resource_instance_by_experiment_instance_id, exp_invalid_type)
exp_invalid_inst = ExperimentInstanceId("exp.invalid", "ud-pld",
"PLD Experiments")
self.assertRaises(
CoordExc.ExperimentNotFoundError, self.resources_manager.
get_resource_instance_by_experiment_instance_id, exp_invalid_inst)
def test_remove_resource_instance_id(self):
session = self.session_maker()
try:
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
experiment_instances = session.query(
CoordinatorModel.ExperimentInstance).all()
self.assertEquals(1, len(experiment_instances))
self.resources_manager.remove_resource_instance_id(session, exp_id)
experiment_instances = session.query(
CoordinatorModel.ExperimentInstance).all()
self.assertEquals(0, len(experiment_instances))
session.commit()
finally:
session.close()
def test_remove_resource_instance(self):
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
resource_instance = Resource("type", "instance")
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id, resource_instance)
# Checking that the resources are there
experiment_instances = self.resources_manager.list_experiment_instances_by_type(
exp_id.to_experiment_id())
self.assertEquals(1, len(experiment_instances))
resource_instances = self.resources_manager.list_resource_instances()
self.assertEquals(1, len(resource_instances))
# Removing resource instance
self.resources_manager.remove_resource_instance(resource_instance)
# Checking that the resources are not there, neither the experiment instances
resource_instances = self.resources_manager.list_resource_instances()
self.assertEquals(0, len(resource_instances))
experiment_instances = self.resources_manager.list_experiment_instances_by_type(
exp_id.to_experiment_id())
self.assertEquals(0, len(experiment_instances))
def test_add_experiment_instance_id(self):
session = self.session_maker()
try:
resource_types = session.query(CoordinatorModel.ResourceType).all()
self.assertEquals(
0, len(resource_types),
"No resource expected in the beginning of the test")
exp_id = ExperimentInstanceId("exp1", "ud-pld", "PLD Experiments")
session.commit()
finally:
session.close()
self.resources_manager.add_experiment_instance_id(
"laboratory1:WL_SERVER1@WL_MACHINE1", exp_id,
Resource("type", "instance"))
session = self.session_maker()
try:
self._check_resource_added(session)
self._check_experiment_instance_id_added(session)
session.commit()
finally:
session.close()
def setUp(self):
self.coord_address = CoordAddress.translate(
"server0:instance0@machine0")
self.mock_locator = self.mocker.mock()
self.cfg_manager = ConfigurationManager.ConfigurationManager()
self.cfg_manager.append_module(configuration_module)
self.cfg_manager._set_value(COORDINATOR_LABORATORY_SERVERS, {
u'lab1:inst@machine': {
'inst1|exp1|cat1': 'res_inst@res_type'
},
})
self.coordinator = coordinator_create(SQLALCHEMY, self.mock_locator,
self.cfg_manager)
self.coordinator._clean()
self.confirmer = self.coordinator.confirmer
self.lab_address = u"lab1:inst@machine"
self.coordinator.add_experiment_instance_id(
self.lab_address, ExperimentInstanceId('inst1', 'exp1', 'cat1'),
Resource("res_type", "res_inst"))
def _update_queues(self):
###########################################################
# There are reasons why a waiting reservation may not be
# able to be promoted while the next one is. For instance,
# if a user is waiting for "pld boards", but only for
# instances of "pld boards" which have a "ud-binary@Binary
# experiments" server running. If only a "ud-pld@PLD
# Experiments" is available, then this user will not be
# promoted and the another user which is waiting for a
# "ud-pld@PLD Experiments" can be promoted.
#
# Therefore, we have a list of the IDs of the waiting
# reservations we previously thought that they couldn't be
# promoted in this iteration. They will have another
# chance in the next run of _update_queues.
#
previously_waiting_reservation_ids = []
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
weblab_resource_slots = WEBLAB_RESOURCE_SLOTS % self.resource_type_name
###########################################################
# While there are free instances and waiting reservations,
# take the first waiting reservation and set it to current
# reservation. Make this repeatedly because we want to
# commit each change
#
while True:
client = self.redis_maker()
filled_waiting_reservation_ids = client.zrangebyscore(
weblab_resource_pqueue_sorted,
-10000,
+10000,
start=0,
num=len(previously_waiting_reservation_ids) + 1)
first_waiting_reservation_id = None
for filled_waiting_reservation_id in filled_waiting_reservation_ids:
waiting_reservation_id = filled_waiting_reservation_id[
filled_waiting_reservation_id.find('_') + 1:]
if waiting_reservation_id not in previously_waiting_reservation_ids:
first_waiting_reservation_id = waiting_reservation_id
break
if first_waiting_reservation_id is None:
return # There is no waiting reservation for this resource that we haven't already tried
previously_waiting_reservation_ids.append(
first_waiting_reservation_id)
#
# For the current resource_type, let's ask for
# all the resource instances available (i.e. those
# who are a member on weblab:resource:%s:slots )
#
free_instances = [
Resource(self.resource_type_name, resource_instance)
for resource_instance in client.smembers(weblab_resource_slots)
]
if len(free_instances) == 0:
# If there is no free instance, just return
return
#
# Select the correct free_instance for the current student among
# all the free_instances
#
if self.randomize_instances:
randomized_free_instances = [
free_instance for free_instance in free_instances
]
random.shuffle(randomized_free_instances)
else:
randomized_free_instances = sorted(
free_instances,
cmp=lambda r1, r2: cmp(r1.resource_type, r2.resource_type)
or cmp(r1.resource_instance, r2.resource_instance))
for free_instance in randomized_free_instances:
#
# IMPORTANT: from here on every "continue" should first revoke the
# reservations_manager and resources_manager confirmations
#
working = self.resources_manager.check_working(free_instance)
if not working:
# The instance is not working
continue
confirmed = self.reservations_manager.confirm(
first_waiting_reservation_id)
if not confirmed:
# student has already been confirmed somewhere else, so don't try with other
# instances, but rather with other student
break
acquired = self.resources_manager.acquire_resource(
free_instance)
# print "ACQUIRED", free_instance, acquired, time.time()
if not acquired:
# the instance has been acquired by someone else. unconfirm student and
# try again with other free_instance
self.reservations_manager.downgrade_confirmation(
first_waiting_reservation_id)
continue
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (
self.resource_type_name, free_instance.resource_instance)
client.sadd(weblab_resource_pqueue_instance_reservations,
first_waiting_reservation_id)
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, first_waiting_reservation_id)
pqueue_reservation_data_str = client.get(
weblab_reservation_pqueue)
reservation_data = self.reservations_manager.get_reservation_data(
first_waiting_reservation_id)
if pqueue_reservation_data_str is None or reservation_data is None:
# the student is not here anymore; downgrading confirmation is not required
# but releasing the resource is; and skip the rest of the free instances
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations,
first_waiting_reservation_id)
break
pqueue_reservation_data = json.loads(
pqueue_reservation_data_str)
start_time = self.time_provider.get_time()
total_time = pqueue_reservation_data[TIME]
pqueue_reservation_data[START_TIME] = start_time
pqueue_reservation_data[TIMESTAMP_BEFORE] = start_time
pqueue_reservation_data[
ACTIVE_STATUS] = STATUS_WAITING_CONFIRMATION
pqueue_reservation_data[
RESOURCE_INSTANCE] = free_instance.to_weblab_str()
initialization_in_accounting = pqueue_reservation_data[
INITIALIZATION_IN_ACCOUNTING]
client_initial_data = reservation_data[CLIENT_INITIAL_DATA]
request_info = json.loads(reservation_data[REQUEST_INFO])
username = request_info.get('username')
locale = request_info.get('locale')
requested_experiment_type = ExperimentId.parse(
reservation_data[EXPERIMENT_TYPE])
selected_experiment_instance = None
experiment_instances = self.resources_manager.list_experiment_instance_ids_by_resource(
free_instance)
for experiment_instance in experiment_instances:
if experiment_instance.to_experiment_id(
) == requested_experiment_type:
selected_experiment_instance = experiment_instance
if selected_experiment_instance is None:
# This resource is not valid for this user, other free_instance should be
# selected. Try with other, but first clean the acquired resources
self.reservations_manager.downgrade_confirmation(
first_waiting_reservation_id)
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations,
first_waiting_reservation_id)
continue
pqueue_reservation_data[
EXPERIMENT_INSTANCE] = selected_experiment_instance.to_weblab_str(
)
laboratory_coord_address = self.resources_manager.get_laboratory_coordaddress_by_experiment_instance_id(
selected_experiment_instance)
pqueue_reservation_data[LAB_COORD] = laboratory_coord_address
client.set(weblab_reservation_pqueue,
json.dumps(pqueue_reservation_data))
filled_reservation_id = client.hget(
weblab_resource_pqueue_map, first_waiting_reservation_id)
client.zrem(weblab_resource_pqueue_sorted,
filled_reservation_id)
#
# Enqueue the confirmation, since it might take a long time
# (for instance, if the laboratory server does not reply because
# of any network problem, or it just takes too much in replying),
# so this method might take too long. That's why we enqueue these
# petitions and run them in other threads.
#
deserialized_server_initial_data = {
'priority.queue.slot.length':
'%s' % total_time,
'priority.queue.slot.start':
'%s' % datetime.datetime.fromtimestamp(start_time),
'priority.queue.slot.initialization_in_accounting':
initialization_in_accounting,
'request.experiment_id.experiment_name':
selected_experiment_instance.exp_name,
'request.experiment_id.category_name':
selected_experiment_instance.cat_name,
'request.username':
username,
'request.full_name':
username,
'request.locale':
locale,
# TODO: add the username and user full name here
}
server_initial_data = json.dumps(
deserialized_server_initial_data)
# server_initial_data will contain information such as "what was the last experiment used?".
# If a single resource was used by a binary experiment, then the next time may not require reprogramming the device
self.confirmer.enqueue_confirmation(
laboratory_coord_address, first_waiting_reservation_id,
selected_experiment_instance, client_initial_data,
server_initial_data, self.resource_type_name)
#
# After it, keep in the while True in order to add the next
# reservation
#
break
def test_list_experiments(self):
self.coordinator.add_experiment_instance_id("server:laboratoryserver@labmachine", ExperimentInstanceId('inst','ud-dummy2','Dummy experiments'), Resource("res_type", "res_inst"))
expected = "ud-dummy@Dummy experiments\n"
expected += "ud-dummy2@Dummy experiments\n"
result = methods.list_experiments.call()
self.assertEquals(expected, result)
