def __init__(self, generic_scheduler_arguments, randomize_instances = True, **kwargs):
super(PriorityQueueScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.randomize_instances = randomize_instances
self._synchronizer = SchedulerTransactionsSynchronizer(self)
self._synchronizer.start()
def __init__(self, generic_scheduler_arguments, randomize_instances = True, **kwargs):
super(PriorityQueueScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.randomize_instances = randomize_instances
self._synchronizer = SchedulerTransactionsSynchronizer(self)
self._synchronizer.start()
class PriorityQueueScheduler(Scheduler):
def __init__(self, generic_scheduler_arguments, randomize_instances = True, **kwargs):
super(PriorityQueueScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.randomize_instances = randomize_instances
self._synchronizer = SchedulerTransactionsSynchronizer(self)
self._synchronizer.start()
@Override(Scheduler)
def stop(self):
self._synchronizer.stop()
@Override(Scheduler)
def is_remote(self):
return False
@exc_checker
@logged()
@Override(Scheduler)
def removing_current_resource_slot(self, session, resource_instance_id):
resource_type = session.query(ResourceType).filter_by(name = resource_instance_id.resource_type).one()
resource_instance = session.query(ResourceInstance).filter_by(name = resource_instance_id.resource_instance, resource_type = resource_type).one()
current_resource_slot = resource_instance.slot
if current_resource_slot is not None:
slot_reservation = current_resource_slot.slot_reservation
if slot_reservation is not None:
concrete_current_reservations = current_resource_slot.slot_reservation.pq_current_reservations
if len(concrete_current_reservations) > 0:
concrete_current_reservation = concrete_current_reservations[0]
waiting_reservation = WaitingReservation(resource_instance.resource_type, concrete_current_reservation.current_reservation_id, concrete_current_reservation.time, -1, concrete_current_reservation.initialization_in_accounting) # -1 : Highest priority
self.reservations_manager.downgrade_confirmation(session, concrete_current_reservation.current_reservation_id)
self.resources_manager.release_resource(session, current_resource_slot.slot_reservation)
session.add(waiting_reservation)
session.delete(concrete_current_reservation)
return True
return False
@exc_checker
@logged()
@Override(Scheduler)
def reserve_experiment(self, reservation_id, experiment_id, time, priority, initialization_in_accounting, client_initial_data, request_info):
"""
priority: the less, the more priority
"""
session = self.session_maker()
try:
resource_type = session.query(ResourceType).filter_by(name = self.resource_type_name).one()
waiting_reservation = WaitingReservation(resource_type, reservation_id, time, priority, initialization_in_accounting)
session.add(waiting_reservation)
session.commit()
finally:
session.close()
return self.get_reservation_status(reservation_id)
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@exc_checker
@logged()
@Override(Scheduler)
def get_reservation_status(self, reservation_id):
self._remove_expired_reservations()
try:
session = self.session_maker()
try:
self.reservations_manager.update(session, reservation_id)
session.commit()
finally:
session.close()
except StaleDataError:
time.sleep(TIME_ANTI_RACE_CONDITIONS * random.random())
return self.get_reservation_status(reservation_id)
self._synchronizer.request_and_wait()
reservation_id_with_route = '%s;%s.%s' % (reservation_id, reservation_id, self.core_server_route)
return_current_status = False
session = self.session_maker()
try:
#
# If the current user is actually in a reservation assigned to a
# certain laboratory, it may be in a Reserved state or in a
# WaitingConfirmation state (meaning that it is still waiting for
# a response from the Laboratory).
#
concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == reservation_id).first()
if concrete_current_reservation is not None:
resource_instance = concrete_current_reservation.slot_reservation.current_resource_slot.resource_instance
requested_experiment_instance = None
for experiment_instance in resource_instance.experiment_instances:
if experiment_instance.experiment_type == concrete_current_reservation.current_reservation.reservation.experiment_type:
requested_experiment_instance = experiment_instance
break
if requested_experiment_instance is None:
raise Exception("Invalid state: there is an resource_instance of the resource_type the user was waiting for which doesn't have any experiment_instance of the experiment_type the user was waiting for")
str_lab_coord_address = requested_experiment_instance.laboratory_coord_address
lab_coord_address = CoordAddress.translate(str_lab_coord_address)
obtained_time = concrete_current_reservation.time
lab_session_id = concrete_current_reservation.lab_session_id
if concrete_current_reservation.exp_info:
exp_info = json.loads(concrete_current_reservation.exp_info)
else:
exp_info = {}
initial_configuration = concrete_current_reservation.initial_configuration
initialization_in_accounting = concrete_current_reservation.initialization_in_accounting
if concrete_current_reservation.timestamp_before is None:
timestamp_before = None
else:
timestamp_before = datetime.datetime.fromtimestamp(concrete_current_reservation.timestamp_before)
if concrete_current_reservation.timestamp_after is None:
timestamp_after = None
else:
timestamp_after = datetime.datetime.fromtimestamp(concrete_current_reservation.timestamp_after)
if lab_session_id is None:
return WSS.WaitingConfirmationQueueStatus(reservation_id_with_route, self.core_server_url)
else:
if initialization_in_accounting:
before = concrete_current_reservation.timestamp_before
else:
before = concrete_current_reservation.timestamp_after
if before is not None:
remaining = (before + obtained_time) - self.time_provider.get_time()
else:
remaining = obtained_time
return WSS.LocalReservedStatus(reservation_id_with_route, lab_coord_address, SessionId.SessionId(lab_session_id), exp_info, obtained_time, initial_configuration, timestamp_before, timestamp_after, initialization_in_accounting, remaining, self.core_server_url)
resource_type = session.query(ResourceType).filter_by(name = self.resource_type_name).one()
waiting_reservation = session.query(WaitingReservation).filter_by(reservation_id = reservation_id, resource_type_id = resource_type.id).first()
if waiting_reservation is None:
waiting_reservations = []
else:
#
# If it has not been assigned to any laboratory, then it might
# be waiting in the queue of that resource type (Waiting) or
# waiting for instances (WaitingInstances, meaning that there is
# no resource of that type implemented)
#
waiting_reservations = session.query(WaitingReservation)\
.filter(WaitingReservation.resource_type == waiting_reservation.resource_type).order_by(WaitingReservation.priority, WaitingReservation.id).all()
if waiting_reservation is None or waiting_reservation not in waiting_reservations:
#
# The position has changed and it is not in the list anymore!
# This has happened using WebLab Bot with 65 users.
#
return_current_status = True
else:
position = waiting_reservations.index(waiting_reservation)
remaining_working_instances = False
for resource_instance in waiting_reservation.resource_type.instances:
if resource_instance.slot is not None:
remaining_working_instances = True
break
finally:
session.close()
if return_current_status:
time.sleep(TIME_ANTI_RACE_CONDITIONS * random.random())
return self.get_reservation_status(reservation_id)
if remaining_working_instances:
return WSS.WaitingQueueStatus(reservation_id_with_route, position)
else:
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route, position)
################################################################
#
# Called when it is confirmed by the Laboratory Server.
#
@exc_checker
@logged()
@Override(Scheduler)
def confirm_experiment(self, reservation_id, lab_session_id, initial_configuration, exp_info):
self._remove_expired_reservations()
session = self.session_maker()
try:
if not self.reservations_manager.check(session, reservation_id):
return
possible_concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == reservation_id).first()
concrete_current_reservation = None
if possible_concrete_current_reservation is not None:
slot = possible_concrete_current_reservation.slot_reservation
if slot is not None:
current_resource_slot = slot.current_resource_slot
if current_resource_slot is not None:
resource_instance = current_resource_slot.resource_instance
if resource_instance is not None:
resource_type = resource_instance.resource_type
if resource_type is not None and resource_type.name == self.resource_type_name:
concrete_current_reservation = possible_concrete_current_reservation
if concrete_current_reservation is None:
return
concrete_current_reservation.lab_session_id = lab_session_id.id
concrete_current_reservation.initial_configuration = initial_configuration
concrete_current_reservation.exp_info = json.dumps(exp_info)
concrete_current_reservation.set_timestamp_after(self.time_provider.get_time())
session.commit()
finally:
session.close()
################################################################
#
# Called when the user disconnects or finishes the resource.
#
@exc_checker
@logged()
@Override(Scheduler)
def finish_reservation(self, reservation_id):
self._remove_expired_reservations()
session = self.session_maker()
try:
possible_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == reservation_id).first()
# Clean current reservation... if the current reservation is assigned to this scheduler
concrete_current_reservation = None
enqueue_free_experiment_args = None
if possible_current_reservation is not None:
slot = possible_current_reservation.slot_reservation
if slot is not None:
current_resource_slot = slot.current_resource_slot
if current_resource_slot is not None:
resource_instance = current_resource_slot.resource_instance
if resource_instance is not None:
resource_type = resource_instance.resource_type
if resource_type is not None and resource_type.name == self.resource_type_name:
concrete_current_reservation = possible_current_reservation
enqueue_free_experiment_args = self._clean_current_reservation(session, concrete_current_reservation)
db_resource_type = session.query(ResourceType).filter_by(name = self.resource_type_name).first()
reservation_to_delete = concrete_current_reservation or session.query(WaitingReservation).filter_by(reservation_id = reservation_id, resource_type = db_resource_type).first()
if reservation_to_delete is not None:
session.delete(reservation_to_delete)
session.commit()
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
finally:
session.close()
def _clean_current_reservation(self, session, concrete_current_reservation):
enqueue_free_experiment_args = None
if concrete_current_reservation is not None:
resource_instance = concrete_current_reservation.slot_reservation.current_resource_slot.resource_instance
if resource_instance is not None: # If the resource instance does not exist anymore, there is no need to call the free_experiment method
lab_session_id = concrete_current_reservation.lab_session_id
experiment_instance = None
for experiment_instance in resource_instance.experiment_instances:
if experiment_instance.experiment_type == concrete_current_reservation.current_reservation.reservation.experiment_type:
experiment_instance = experiment_instance
break
if experiment_instance is not None: # If the experiment instance doesn't exist, there is no need to call the free_experiment method
lab_coord_address = experiment_instance.laboratory_coord_address
reservation_id = concrete_current_reservation.current_reservation_id
enqueue_free_experiment_args = (lab_coord_address, reservation_id, lab_session_id, experiment_instance.to_experiment_instance_id())
self.reservations_manager.downgrade_confirmation(session, concrete_current_reservation.current_reservation_id)
return enqueue_free_experiment_args
def update(self):
self._update_queues()
#############################################################
#
# Take the queue of a given Resource Type and update it
#
@exc_checker
def _update_queues(self):
###########################################################
# There are reasons why a waiting reservation may not be
# able to be promoted while the next one is. For instance,
# if a user is waiting for "pld boards", but only for
# instances of "pld boards" which have a "ud-binary@Binary
# experiments" server running. If only a "ud-pld@PLD
# Experiments" is available, then this user will not be
# promoted and the another user which is waiting for a
# "ud-pld@PLD Experiments" can be promoted.
#
# Therefore, we have a list of the IDs of the waiting
# reservations we previously thought that they couldn't be
# promoted in this iteration. They will have another
# chance in the next run of _update_queues.
#
previously_waiting_reservation_ids = []
###########################################################
# While there are free instances and waiting reservations,
# take the first waiting reservation and set it to current
# reservation. Make this repeatedly because we want to
# commit each change
#
while True:
session = self.session_maker()
try:
resource_type = session.query(ResourceType).filter(ResourceType.name == self.resource_type_name).first()
#
# Retrieve the first waiting reservation. If there is no one that
# we haven't tried already, return
#
first_waiting_reservations = session.query(WaitingReservation).filter(WaitingReservation.resource_type == resource_type).order_by(WaitingReservation.priority, WaitingReservation.id)[:len(previously_waiting_reservation_ids) + 1]
first_waiting_reservation = None
for waiting_reservation in first_waiting_reservations:
if waiting_reservation.id not in previously_waiting_reservation_ids:
first_waiting_reservation = waiting_reservation
break
if first_waiting_reservation is None:
return # There is no waiting reservation for this resource that we haven't already tried
previously_waiting_reservation_ids.append(first_waiting_reservation.id)
#
# For the current resource_type, let's ask for
# all the resource instances available (i.e. those
# who have no SchedulingSchemaIndependentSlotReservation
# associated)
#
free_instances = session.query(CurrentResourceSlot)\
.select_from(join(CurrentResourceSlot, ResourceInstance))\
.filter(not_(CurrentResourceSlot.slot_reservations.any()))\
.filter(ResourceInstance.resource_type == resource_type)\
.order_by(CurrentResourceSlot.id).all()
if len(free_instances) == 0:
# If there is no free instance, just return
return
#
# Select the correct free_instance for the current student among
# all the free_instances
#
if self.randomize_instances:
randomized_free_instances = [ free_instance for free_instance in free_instances ]
random.shuffle(randomized_free_instances)
else:
randomized_free_instances = free_instances
for free_instance in randomized_free_instances:
resource_type = free_instance.resource_instance.resource_type
if resource_type is None:
continue # If suddenly the free_instance is not a free_instance anymore, try with other free_instance
#
# IMPORTANT: from here on every "continue" should first revoke the
# reservations_manager and resources_manager confirmations
#
self.reservations_manager.confirm(session, first_waiting_reservation.reservation_id)
slot_reservation = self.resources_manager.acquire_resource(session, free_instance)
total_time = first_waiting_reservation.time
initialization_in_accounting = first_waiting_reservation.initialization_in_accounting
start_time = self.time_provider.get_time()
concrete_current_reservation = ConcreteCurrentReservation(slot_reservation, first_waiting_reservation.reservation_id,
total_time, start_time, first_waiting_reservation.priority, first_waiting_reservation.initialization_in_accounting)
concrete_current_reservation.set_timestamp_before(self.time_provider.get_time())
client_initial_data = first_waiting_reservation.reservation.client_initial_data
request_info = json.loads(first_waiting_reservation.reservation.request_info)
username = request_info.get('username')
username_unique = request_info.get('username_unique')
locale = request_info.get('locale')
reservation_id = first_waiting_reservation.reservation_id
if reservation_id is None:
break # If suddenly the waiting_reservation is not a waiting_reservation anymore, so reservation is None, go again to the while True.
requested_experiment_type = first_waiting_reservation.reservation.experiment_type
selected_experiment_instance = None
for experiment_instance in free_instance.resource_instance.experiment_instances:
if experiment_instance.experiment_type == requested_experiment_type:
selected_experiment_instance = experiment_instance
if selected_experiment_instance is None:
# This resource is not valid for this user, other free_instance should be
# selected. Try with other, but first clean the acquired resources
self.reservations_manager.downgrade_confirmation(session, first_waiting_reservation.reservation_id)
self.resources_manager.release_resource(session, slot_reservation)
continue
experiment_instance_id = ExperimentInstanceId(selected_experiment_instance.experiment_instance_id, requested_experiment_type.exp_name, requested_experiment_type.cat_name)
laboratory_coord_address = selected_experiment_instance.laboratory_coord_address
try:
session.delete(first_waiting_reservation)
session.add(concrete_current_reservation)
session.commit()
except IntegrityError as ie:
if DEBUG:
print("IntegrityError when adding concrete_current_reservation: ", sys.exc_info())
# Other scheduler confirmed the user or booked the reservation, rollback and try again
# But log just in case
log.log(
PriorityQueueScheduler, log.level.Warning,
"IntegrityError looping on update_queues: %s" % ie )
log.log_exc(PriorityQueueScheduler, log.level.Info)
session.rollback()
break
except Exception as e:
if DEBUG:
print("Other error when adding concrete_current_reservation: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"Exception looping on update_queues: %s" % e )
log.log_exc(PriorityQueueScheduler, log.level.Info)
session.rollback()
break
else:
#
# Enqueue the confirmation, since it might take a long time
# (for instance, if the laboratory server does not reply because
# of any network problem, or it just takes too much in replying),
# so this method might take too long. That's why we enqueue these
# petitions and run them in other threads.
#
timezone = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone
deserialized_server_initial_data = {
'priority.queue.slot.length' : '%s' % total_time,
'priority.queue.slot.start' : '%s' % datetime.datetime.fromtimestamp(start_time),
'priority.queue.slot.start.utc' : '%s' % datetime.datetime.utcfromtimestamp(start_time),
'priority.queue.slot.start.timestamp' : '%s' % start_time,
'priority.queue.slot.start.timezone' : '%s' % timezone,
'priority.queue.slot.initialization_in_accounting' : initialization_in_accounting,
'request.experiment_id.experiment_name' : experiment_instance_id.exp_name,
'request.experiment_id.category_name' : experiment_instance_id.cat_name,
'request.username' : username,
'request.username.unique' : username_unique,
'request.full_name' : username,
'request.locale' : locale,
'weblab_reservation_id' : first_waiting_reservation.reservation_id,
}
server_initial_data = json.dumps(deserialized_server_initial_data)
# server_initial_data will contain information such as "what was the last experiment used?".
# If a single resource was used by a binary experiment, then the next time may not require reprogramming the device
self.confirmer.enqueue_confirmation(laboratory_coord_address, reservation_id, experiment_instance_id, client_initial_data, server_initial_data, self.resource_type_name)
#
# After it, keep in the while True in order to add the next
# reservation
#
break
except (ConcurrentModificationError, IntegrityError) as ie:
# Something happened somewhere else, such as the user being confirmed twice, the experiment being reserved twice or so on.
# Rollback and start again
if DEBUG:
print("Other ConcurrentModificationError or IntegrityError in update_queues: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"Exception while updating queues, reverting and trying again: %s" % ie )
log.log_exc(PriorityQueueScheduler, log.level.Info)
session.rollback()
finally:
session.close()
################################################
#
# Remove all reservations whose session has expired
#
def _remove_expired_reservations(self):
session = self.session_maker()
try:
now = self.time_provider.get_time()
current_expiration_time = datetime.datetime.utcfromtimestamp(now - EXPIRATION_TIME)
reservations_removed = False
enqueue_free_experiment_args_retrieved = []
expired_query = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.expired_timestamp != 0).filter(ConcreteCurrentReservation.expired_timestamp < self.time_provider.get_time())
for expired_concrete_current_reservation in expired_query.all():
expired_reservation = expired_concrete_current_reservation.current_reservation_id
if expired_reservation is None:
continue # Maybe it's not an expired_reservation anymore
enqueue_free_experiment_args = self._clean_current_reservation(session, expired_concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(expired_concrete_current_reservation)
self.reservations_manager.delete(session, expired_reservation)
reservations_removed = True
for expired_reservation_id in self.reservations_manager.list_expired_reservations(session, current_expiration_time):
concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == expired_reservation_id).first()
if concrete_current_reservation is not None:
enqueue_free_experiment_args = self._clean_current_reservation(session, concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(concrete_current_reservation)
waiting_reservation = session.query(WaitingReservation).filter(WaitingReservation.reservation_id == expired_reservation_id).first()
if waiting_reservation is not None:
session.delete(waiting_reservation)
self.reservations_manager.delete(session, expired_reservation_id)
reservations_removed = True
if reservations_removed:
try:
session.commit()
except ConcurrentModificationError as e:
if DEBUG:
print("Other error when commiting when reservations_removed: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"IntegrityError: %s" % e )
log.log_exc(PriorityQueueScheduler, log.level.Info)
pass # Someone else removed these users before us.
else:
for enqueue_free_experiment_args in enqueue_free_experiment_args_retrieved:
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
else:
session.rollback()
finally:
session.close()
##############################################################
#
# ONLY FOR TESTING: It completely removes the whole database
#
@Override(Scheduler)
def _clean(self):
session = self.session_maker()
try:
for waiting_reservation in session.query(WaitingReservation).all():
session.delete(waiting_reservation)
for concrete_current_reservation in session.query(ConcreteCurrentReservation).all():
session.delete(concrete_current_reservation)
for current_resource_slot in session.query(CurrentResourceSlot).all():
session.delete(current_resource_slot)
session.commit()
except ConcurrentModificationError:
if DEBUG:
print("Error when cleaning: ", sys.exc_info())
pass # Another process is cleaning concurrently
finally:
session.close()
class PriorityQueueScheduler(Scheduler):
def __init__(self,
generic_scheduler_arguments,
randomize_instances=True,
**kwargs):
super(PriorityQueueScheduler,
self).__init__(generic_scheduler_arguments, **kwargs)
self.randomize_instances = randomize_instances
self._synchronizer = SchedulerTransactionsSynchronizer(self)
self._synchronizer.start()
@Override(Scheduler)
def stop(self):
self._synchronizer.stop()
@Override(Scheduler)
def is_remote(self):
return False
@exc_checker
@logged()
@Override(Scheduler)
@typecheck(typecheck.ANY, typecheck.ANY, Resource)
def removing_current_resource_slot(self, client, resource):
weblab_resource_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (
resource.resource_type, resource.resource_instance)
current_reservation_ids = client.smembers(
weblab_resource_instance_reservations)
if len(current_reservation_ids) > 0:
current_reservation_id = list(current_reservation_ids)[0]
if client.srem(weblab_resource_instance_reservations,
current_reservation_id):
self.reservations_manager.downgrade_confirmation(
current_reservation_id)
self.resources_manager.release_resource(resource)
# Remove data that was added when confirmed
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, current_reservation_id)
reservation_data_str = client.get(weblab_reservation_pqueue)
reservation_data = json.loads(reservation_data_str)
reservation_data.pop(ACTIVE_STATUS, None)
reservation_data.pop(TIMESTAMP_BEFORE, None)
reservation_data.pop(TIMESTAMP_AFTER, None)
reservation_data.pop(LAB_SESSION_ID, None)
reservation_data.pop(EXP_INFO, None)
reservation_data_str = json.dumps(reservation_data)
reservation_data = client.set(weblab_reservation_pqueue,
reservation_data_str)
# Add back to the queue
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
filled_reservation_id = client.hget(weblab_resource_pqueue_map,
current_reservation_id)
client.zadd(weblab_resource_pqueue_sorted,
filled_reservation_id, -1)
return True
return False
@exc_checker
@logged()
@Override(Scheduler)
def reserve_experiment(self, reservation_id, experiment_id, time, priority,
initialization_in_accounting, client_initial_data,
request_info):
"""
priority: the less, the more priority
"""
client = self.redis_maker()
# For indexing purposes
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, reservation_id)
weblab_resource_reservations = WEBLAB_RESOURCE_RESERVATIONS % self.resource_type_name
# Queue management
weblab_resource_pqueue_reservations = WEBLAB_RESOURCE_PQUEUE_RESERVATIONS % self.resource_type_name
weblab_resource_pqueue_positions = WEBLAB_RESOURCE_PQUEUE_POSITIONS % self.resource_type_name
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
# Within the same priority, we want all to sort all the requests by the order they came.
# In order to support this, we increase a long enough value and put it before the reservaiton_id
current_position = client.incr(weblab_resource_pqueue_positions)
filled_reservation_id = "%s_%s" % (str(current_position).zfill(100),
reservation_id)
pipeline = client.pipeline()
pipeline.hset(weblab_resource_pqueue_map, reservation_id,
filled_reservation_id)
pipeline.zadd(weblab_resource_pqueue_sorted, filled_reservation_id,
priority)
pipeline.sadd(weblab_resource_reservations, reservation_id)
pipeline.sadd(weblab_resource_pqueue_reservations, reservation_id)
generic_data = {
TIME: time,
INITIALIZATION_IN_ACCOUNTING: initialization_in_accounting,
PRIORITY: priority,
}
pipeline.set(weblab_reservation_pqueue, json.dumps(generic_data))
pipeline.execute()
return self.get_reservation_status(reservation_id)
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@exc_checker
@logged()
@Override(Scheduler)
def get_reservation_status(self, reservation_id):
self._remove_expired_reservations()
expired = self.reservations_manager.update(reservation_id)
if expired:
self._delete_reservation(reservation_id)
raise ExpiredSessionError("Expired reservation")
self._synchronizer.request_and_wait()
reservation_id_with_route = '%s;%s.%s' % (
reservation_id, reservation_id, self.core_server_route)
client = self.redis_maker()
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, reservation_id)
reservation_data_str = client.get(weblab_reservation_pqueue)
if reservation_data_str is None:
log.log(
PriorityQueueScheduler, log.level.Error,
"get_reservation_status called with a reservation_id that is not registered (not found on weblab_reservation_pqueue). Returning a WaitingInstanceStatus"
)
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route,
50)
reservation_data = json.loads(reservation_data_str)
if ACTIVE_STATUS in reservation_data:
# Reserved or Waiting reservation
status = reservation_data[ACTIVE_STATUS]
# It may be just waiting for the experiment server to respond
if status == STATUS_WAITING_CONFIRMATION:
return WSS.WaitingConfirmationQueueStatus(
reservation_id_with_route, self.core_server_url)
# Or the experiment server already responded and therefore we have all this data
str_lab_coord_address = reservation_data[LAB_COORD]
obtained_time = reservation_data[TIME]
initialization_in_accounting = reservation_data[
INITIALIZATION_IN_ACCOUNTING]
lab_session_id = reservation_data[LAB_SESSION_ID]
initial_configuration = reservation_data[INITIAL_CONFIGURATION]
timestamp_before_tstamp = reservation_data[TIMESTAMP_BEFORE]
timestamp_after_tstamp = reservation_data[TIMESTAMP_AFTER]
if EXP_INFO in reservation_data and reservation_data[EXP_INFO]:
exp_info = json.loads(reservation_data[EXP_INFO])
else:
exp_info = {}
timestamp_before = datetime.datetime.fromtimestamp(
timestamp_before_tstamp)
timestamp_after = datetime.datetime.fromtimestamp(
timestamp_after_tstamp)
lab_coord_address = CoordAddress.translate(str_lab_coord_address)
if initialization_in_accounting:
before = timestamp_before_tstamp
else:
before = timestamp_after_tstamp
if before is not None:
remaining = (before +
obtained_time) - self.time_provider.get_time()
else:
remaining = obtained_time
return WSS.LocalReservedStatus(
reservation_id_with_route, lab_coord_address,
SessionId.SessionId(lab_session_id), exp_info, obtained_time,
initial_configuration, timestamp_before, timestamp_after,
initialization_in_accounting, remaining, self.core_server_url)
# else it's waiting
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
filled_reservation_id = client.hget(weblab_resource_pqueue_map,
reservation_id)
if filled_reservation_id is None:
log.log(
PriorityQueueScheduler, log.level.Error,
"get_reservation_status called with a reservation_id that is not registered (not found on the reservations map). Returning a WaitingInstanceStatus"
)
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route,
50)
position = client.zrank(weblab_resource_pqueue_sorted,
filled_reservation_id)
if position is None: # It's not in the queue now
time.sleep(TIME_ANTI_RACE_CONDITIONS * random.random())
return self.get_reservation_status(reservation_id)
if self.resources_manager.are_resource_instances_working(
self.resource_type_name):
return WSS.WaitingQueueStatus(reservation_id_with_route, position)
else:
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route,
position)
################################################################
#
# Called when it is confirmed by the Laboratory Server.
#
@exc_checker
@logged()
@Override(Scheduler)
def confirm_experiment(self, reservation_id, lab_session_id,
initial_configuration, exp_info):
self._remove_expired_reservations()
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, reservation_id)
client = self.redis_maker()
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
if pqueue_reservation_data_str is None:
return
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
resource_instance_str = pqueue_reservation_data.get(RESOURCE_INSTANCE)
if resource_instance_str is not None:
resource_instance = Resource.parse(resource_instance_str)
if not self.resources_manager.check_working(resource_instance):
# TODO: if the experiment is broken and the student is ACTIVE_STATUS, something should be done
#
return
pqueue_reservation_data[LAB_SESSION_ID] = lab_session_id.id
pqueue_reservation_data[INITIAL_CONFIGURATION] = initial_configuration
pqueue_reservation_data[TIMESTAMP_AFTER] = self.time_provider.get_time(
)
pqueue_reservation_data[ACTIVE_STATUS] = STATUS_RESERVED
pqueue_reservation_data[EXP_INFO] = json.dumps(exp_info)
pqueue_reservation_data_str = json.dumps(pqueue_reservation_data)
client.set(weblab_reservation_pqueue, pqueue_reservation_data_str)
################################################################
#
# Called when the user disconnects or finishes the resource.
#
@exc_checker
@logged()
@Override(Scheduler)
def finish_reservation(self, reservation_id):
self._remove_expired_reservations()
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, reservation_id)
client = self.redis_maker()
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
if pqueue_reservation_data_str is None:
return
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
if ACTIVE_STATUS in pqueue_reservation_data:
enqueue_free_experiment_args = self._clean_current_reservation(
reservation_id)
else:
enqueue_free_experiment_args = None
self._delete_reservation(reservation_id)
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(
*enqueue_free_experiment_args)
def _clean_current_reservation(self, reservation_id):
client = self.redis_maker()
enqueue_free_experiment_args = None
if reservation_id is not None:
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, reservation_id)
reservation_data_str = client.get(weblab_reservation_pqueue)
if reservation_data_str is not None:
downgraded = self.reservations_manager.downgrade_confirmation(
reservation_id)
if downgraded:
reservation_data = json.loads(reservation_data_str)
resource_instance_str = reservation_data.get(
RESOURCE_INSTANCE)
if resource_instance_str is not None:
resource_instance = Resource.parse(
resource_instance_str)
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (
resource_instance.resource_type,
resource_instance.resource_instance)
client.srem(
weblab_resource_pqueue_instance_reservations,
reservation_id)
# print "RELEASING AT _clean_current_reservation. SHOULD NEVER HAPPEN."
# self.resources_manager.release_resource(resource_instance)
lab_session_id = reservation_data.get(LAB_SESSION_ID)
experiment_instance_str = reservation_data.get(
EXPERIMENT_INSTANCE)
experiment_instance_id = ExperimentInstanceId.parse(
experiment_instance_str)
if experiment_instance_id is not None:
# If the experiment instance doesn't exist, there is no need to call the free_experiment method
lab_coord_address = reservation_data.get(LAB_COORD)
enqueue_free_experiment_args = (
lab_coord_address, reservation_id,
lab_session_id, experiment_instance_id)
# otherwise the student has been removed
return enqueue_free_experiment_args
def update(self):
self._update_queues()
#############################################################
#
# Take the queue of a given Resource Type and update it
#
@exc_checker
def _update_queues(self):
###########################################################
# There are reasons why a waiting reservation may not be
# able to be promoted while the next one is. For instance,
# if a user is waiting for "pld boards", but only for
# instances of "pld boards" which have a "ud-binary@Binary
# experiments" server running. If only a "ud-pld@PLD
# Experiments" is available, then this user will not be
# promoted and the another user which is waiting for a
# "ud-pld@PLD Experiments" can be promoted.
#
# Therefore, we have a list of the IDs of the waiting
# reservations we previously thought that they couldn't be
# promoted in this iteration. They will have another
# chance in the next run of _update_queues.
#
previously_waiting_reservation_ids = []
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
weblab_resource_slots = WEBLAB_RESOURCE_SLOTS % self.resource_type_name
###########################################################
# While there are free instances and waiting reservations,
# take the first waiting reservation and set it to current
# reservation. Make this repeatedly because we want to
# commit each change
#
while True:
client = self.redis_maker()
filled_waiting_reservation_ids = client.zrangebyscore(
weblab_resource_pqueue_sorted,
-10000,
+10000,
start=0,
num=len(previously_waiting_reservation_ids) + 1)
first_waiting_reservation_id = None
for filled_waiting_reservation_id in filled_waiting_reservation_ids:
waiting_reservation_id = filled_waiting_reservation_id[
filled_waiting_reservation_id.find('_') + 1:]
if waiting_reservation_id not in previously_waiting_reservation_ids:
first_waiting_reservation_id = waiting_reservation_id
break
if first_waiting_reservation_id is None:
return # There is no waiting reservation for this resource that we haven't already tried
previously_waiting_reservation_ids.append(
first_waiting_reservation_id)
#
# For the current resource_type, let's ask for
# all the resource instances available (i.e. those
# who are a member on weblab:resource:%s:slots )
#
free_instances = [
Resource(self.resource_type_name, resource_instance)
for resource_instance in client.smembers(weblab_resource_slots)
]
if len(free_instances) == 0:
# If there is no free instance, just return
return
#
# Select the correct free_instance for the current student among
# all the free_instances
#
if self.randomize_instances:
randomized_free_instances = [
free_instance for free_instance in free_instances
]
random.shuffle(randomized_free_instances)
else:
randomized_free_instances = sorted(
free_instances,
cmp=lambda r1, r2: cmp(r1.resource_type, r2.resource_type)
or cmp(r1.resource_instance, r2.resource_instance))
for free_instance in randomized_free_instances:
#
# IMPORTANT: from here on every "continue" should first revoke the
# reservations_manager and resources_manager confirmations
#
working = self.resources_manager.check_working(free_instance)
if not working:
# The instance is not working
continue
confirmed = self.reservations_manager.confirm(
first_waiting_reservation_id)
if not confirmed:
# student has already been confirmed somewhere else, so don't try with other
# instances, but rather with other student
break
acquired = self.resources_manager.acquire_resource(
free_instance)
# print "ACQUIRED", free_instance, acquired, time.time()
if not acquired:
# the instance has been acquired by someone else. unconfirm student and
# try again with other free_instance
self.reservations_manager.downgrade_confirmation(
first_waiting_reservation_id)
continue
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (
self.resource_type_name, free_instance.resource_instance)
client.sadd(weblab_resource_pqueue_instance_reservations,
first_waiting_reservation_id)
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, first_waiting_reservation_id)
pqueue_reservation_data_str = client.get(
weblab_reservation_pqueue)
reservation_data = self.reservations_manager.get_reservation_data(
first_waiting_reservation_id)
if pqueue_reservation_data_str is None or reservation_data is None:
# the student is not here anymore; downgrading confirmation is not required
# but releasing the resource is; and skip the rest of the free instances
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations,
first_waiting_reservation_id)
break
pqueue_reservation_data = json.loads(
pqueue_reservation_data_str)
start_time = self.time_provider.get_time()
total_time = pqueue_reservation_data[TIME]
pqueue_reservation_data[START_TIME] = start_time
pqueue_reservation_data[TIMESTAMP_BEFORE] = start_time
pqueue_reservation_data[
ACTIVE_STATUS] = STATUS_WAITING_CONFIRMATION
pqueue_reservation_data[
RESOURCE_INSTANCE] = free_instance.to_weblab_str()
initialization_in_accounting = pqueue_reservation_data[
INITIALIZATION_IN_ACCOUNTING]
client_initial_data = reservation_data[CLIENT_INITIAL_DATA]
request_info = json.loads(reservation_data[REQUEST_INFO])
username = request_info.get('username')
locale = request_info.get('locale')
requested_experiment_type = ExperimentId.parse(
reservation_data[EXPERIMENT_TYPE])
selected_experiment_instance = None
experiment_instances = self.resources_manager.list_experiment_instance_ids_by_resource(
free_instance)
for experiment_instance in experiment_instances:
if experiment_instance.to_experiment_id(
) == requested_experiment_type:
selected_experiment_instance = experiment_instance
if selected_experiment_instance is None:
# This resource is not valid for this user, other free_instance should be
# selected. Try with other, but first clean the acquired resources
self.reservations_manager.downgrade_confirmation(
first_waiting_reservation_id)
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations,
first_waiting_reservation_id)
continue
pqueue_reservation_data[
EXPERIMENT_INSTANCE] = selected_experiment_instance.to_weblab_str(
)
laboratory_coord_address = self.resources_manager.get_laboratory_coordaddress_by_experiment_instance_id(
selected_experiment_instance)
pqueue_reservation_data[LAB_COORD] = laboratory_coord_address
client.set(weblab_reservation_pqueue,
json.dumps(pqueue_reservation_data))
filled_reservation_id = client.hget(
weblab_resource_pqueue_map, first_waiting_reservation_id)
client.zrem(weblab_resource_pqueue_sorted,
filled_reservation_id)
#
# Enqueue the confirmation, since it might take a long time
# (for instance, if the laboratory server does not reply because
# of any network problem, or it just takes too much in replying),
# so this method might take too long. That's why we enqueue these
# petitions and run them in other threads.
#
deserialized_server_initial_data = {
'priority.queue.slot.length':
'%s' % total_time,
'priority.queue.slot.start':
'%s' % datetime.datetime.fromtimestamp(start_time),
'priority.queue.slot.initialization_in_accounting':
initialization_in_accounting,
'request.experiment_id.experiment_name':
selected_experiment_instance.exp_name,
'request.experiment_id.category_name':
selected_experiment_instance.cat_name,
'request.username':
username,
'request.full_name':
username,
'request.locale':
locale,
# TODO: add the username and user full name here
}
server_initial_data = json.dumps(
deserialized_server_initial_data)
# server_initial_data will contain information such as "what was the last experiment used?".
# If a single resource was used by a binary experiment, then the next time may not require reprogramming the device
self.confirmer.enqueue_confirmation(
laboratory_coord_address, first_waiting_reservation_id,
selected_experiment_instance, client_initial_data,
server_initial_data, self.resource_type_name)
#
# After it, keep in the while True in order to add the next
# reservation
#
break
################################################
#
# Remove all reservations whose session has expired
#
@exc_checker
def _remove_expired_reservations(self):
now = self.time_provider.get_time()
enqueue_free_experiment_args_retrieved = []
client = self.redis_maker()
weblab_resource_pqueue_reservations = WEBLAB_RESOURCE_PQUEUE_RESERVATIONS % self.resource_type_name
reservations = [
reservation_id for reservation_id in client.smembers(
weblab_resource_pqueue_reservations)
]
# Since there might be a lot of reservations, create a pipeline and retrieve
# every reservation data in a row
pipeline = client.pipeline()
for reservation_id in reservations:
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, reservation_id)
pipeline.get(weblab_reservation_pqueue)
results = pipeline.execute()
for reservation_id, reservation_data in zip(reservations, results):
if reservation_data is not None:
data = json.loads(reservation_data)
if ACTIVE_STATUS in data:
total_time = data[TIME]
timestamp_before = data[TIMESTAMP_BEFORE]
timestamp_after = data.get(TIMESTAMP_AFTER)
initialization_in_accounting = data[
INITIALIZATION_IN_ACCOUNTING]
# if timestamp_after is None and initialization should not be considered,
# then we can not calculate if the time has expired, so we skip it (it will
# be considered as expired for lack of LATEST_ACCESS
if timestamp_after is not None or initialization_in_accounting:
timestamp = timestamp_before if initialization_in_accounting else timestamp_after
if now >= timestamp + total_time: # Expired
enqueue_free_experiment_args = self._clean_current_reservation(
reservation_id)
enqueue_free_experiment_args_retrieved.append(
enqueue_free_experiment_args)
self._delete_reservation(reservation_id)
self.reservations_manager.delete(reservation_id)
# Anybody with latest_access later than this point is expired
current_expiration_time = datetime.datetime.utcfromtimestamp(
now - EXPIRATION_TIME)
for expired_reservation_id in self.reservations_manager.list_expired_reservations(
current_expiration_time):
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, expired_reservation_id)
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
if pqueue_reservation_data_str is None:
continue
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
if ACTIVE_STATUS in pqueue_reservation_data:
enqueue_free_experiment_args = self._clean_current_reservation(
expired_reservation_id)
enqueue_free_experiment_args_retrieved.append(
enqueue_free_experiment_args)
self._delete_reservation(expired_reservation_id)
self.reservations_manager.delete(expired_reservation_id)
for enqueue_free_experiment_args in enqueue_free_experiment_args_retrieved:
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(
*enqueue_free_experiment_args)
def _delete_reservation(self, reservation_id):
weblab_resource_pqueue_reservations = WEBLAB_RESOURCE_PQUEUE_RESERVATIONS % self.resource_type_name
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (
self.resource_type_name, reservation_id)
resource_instances = self.resources_manager.list_resource_instances_by_type(
self.resource_type_name)
client = self.redis_maker()
pipeline = client.pipeline()
for resource_instance in resource_instances:
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (
self.resource_type_name, resource_instance.resource_instance)
pipeline.srem(weblab_resource_pqueue_instance_reservations,
reservation_id)
pipeline.srem(weblab_resource_pqueue_reservations, reservation_id)
pipeline.delete(weblab_reservation_pqueue)
pipeline.execute()
filled_reservation_id = client.hget(weblab_resource_pqueue_map,
reservation_id)
client.hdel(weblab_resource_pqueue_map, reservation_id)
client.zrem(weblab_resource_pqueue_sorted, filled_reservation_id)
##############################################################
#
# ONLY FOR TESTING: It completely removes the whole database
#
@Override(Scheduler)
def _clean(self):
client = self.redis_maker()
for reservation_id in self.reservations_manager.list_all_reservations(
):
client.delete(WEBLAB_RESOURCE_RESERVATION_PQUEUE %
(self.resource_type_name, reservation_id))
client.delete(WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS %
(self.resource_type_name, '*'))
for resource_instance in self.resources_manager.list_resource_instances_by_type(
self.resource_type_name):
client.delete(
WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS %
(self.resource_type_name, resource_instance.resource_instance))
client.delete(WEBLAB_RESOURCE_PQUEUE_RESERVATIONS %
self.resource_type_name)
client.delete(WEBLAB_RESOURCE_PQUEUE_POSITIONS %
self.resource_type_name)
client.delete(WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name)
client.delete(WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name)
class PriorityQueueScheduler(Scheduler):
def __init__(self, generic_scheduler_arguments, randomize_instances = True, **kwargs):
super(PriorityQueueScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.randomize_instances = randomize_instances
self._synchronizer = SchedulerTransactionsSynchronizer(self)
self._synchronizer.start()
@Override(Scheduler)
def stop(self):
self._synchronizer.stop()
@Override(Scheduler)
def is_remote(self):
return False
@exc_checker
@logged()
@Override(Scheduler)
def removing_current_resource_slot(self, session, resource_instance_id):
resource_type = session.query(ResourceType).filter_by(name = resource_instance_id.resource_type).one()
resource_instance = session.query(ResourceInstance).filter_by(name = resource_instance_id.resource_instance, resource_type = resource_type).one()
current_resource_slot = resource_instance.slot
if current_resource_slot is not None:
slot_reservation = current_resource_slot.slot_reservation
if slot_reservation is not None:
concrete_current_reservations = current_resource_slot.slot_reservation.pq_current_reservations
if len(concrete_current_reservations) > 0:
concrete_current_reservation = concrete_current_reservations[0]
waiting_reservation = WaitingReservation(resource_instance.resource_type, concrete_current_reservation.current_reservation_id, concrete_current_reservation.time, -1, concrete_current_reservation.initialization_in_accounting) # -1 : Highest priority
self.reservations_manager.downgrade_confirmation(session, concrete_current_reservation.current_reservation_id)
self.resources_manager.release_resource(session, current_resource_slot.slot_reservation)
session.add(waiting_reservation)
session.delete(concrete_current_reservation)
return True
return False
@exc_checker
@logged()
@Override(Scheduler)
def reserve_experiment(self, reservation_id, experiment_id, time, priority, initialization_in_accounting, client_initial_data, request_info):
"""
priority: the less, the more priority
"""
session = self.session_maker()
try:
resource_type = session.query(ResourceType).filter_by(name = self.resource_type_name).one()
waiting_reservation = WaitingReservation(resource_type, reservation_id, time, priority, initialization_in_accounting)
session.add(waiting_reservation)
session.commit()
finally:
session.close()
return self.get_reservation_status(reservation_id)
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@exc_checker
@logged()
@Override(Scheduler)
def get_reservation_status(self, reservation_id):
self._remove_expired_reservations()
try:
session = self.session_maker()
try:
self.reservations_manager.update(session, reservation_id)
session.commit()
finally:
session.close()
except StaleDataError:
time.sleep(TIME_ANTI_RACE_CONDITIONS * random.random())
return self.get_reservation_status(reservation_id)
self._synchronizer.request_and_wait()
reservation_id_with_route = '%s;%s.%s' % (reservation_id, reservation_id, self.core_server_route)
return_current_status = False
session = self.session_maker()
try:
#
# If the current user is actually in a reservation assigned to a
# certain laboratory, it may be in a Reserved state or in a
# WaitingConfirmation state (meaning that it is still waiting for
# a response from the Laboratory).
#
concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == reservation_id).first()
if concrete_current_reservation is not None:
resource_instance = concrete_current_reservation.slot_reservation.current_resource_slot.resource_instance
requested_experiment_instance = None
for experiment_instance in resource_instance.experiment_instances:
if experiment_instance.experiment_type == concrete_current_reservation.current_reservation.reservation.experiment_type:
requested_experiment_instance = experiment_instance
break
if requested_experiment_instance is None:
raise Exception("Invalid state: there is an resource_instance of the resource_type the user was waiting for which doesn't have any experiment_instance of the experiment_type the user was waiting for")
str_lab_coord_address = requested_experiment_instance.laboratory_coord_address
lab_coord_address = CoordAddress.translate(str_lab_coord_address)
obtained_time = concrete_current_reservation.time
lab_session_id = concrete_current_reservation.lab_session_id
if concrete_current_reservation.exp_info:
exp_info = json.loads(concrete_current_reservation.exp_info)
else:
exp_info = {}
initial_configuration = concrete_current_reservation.initial_configuration
initialization_in_accounting = concrete_current_reservation.initialization_in_accounting
if concrete_current_reservation.timestamp_before is None:
timestamp_before = None
else:
timestamp_before = datetime.datetime.fromtimestamp(concrete_current_reservation.timestamp_before)
if concrete_current_reservation.timestamp_after is None:
timestamp_after = None
else:
timestamp_after = datetime.datetime.fromtimestamp(concrete_current_reservation.timestamp_after)
if lab_session_id is None:
return WSS.WaitingConfirmationQueueStatus(reservation_id_with_route, self.core_server_url)
else:
if initialization_in_accounting:
before = concrete_current_reservation.timestamp_before
else:
before = concrete_current_reservation.timestamp_after
if before is not None:
remaining = (before + obtained_time) - self.time_provider.get_time()
else:
remaining = obtained_time
return WSS.LocalReservedStatus(reservation_id_with_route, lab_coord_address, SessionId.SessionId(lab_session_id), exp_info, obtained_time, initial_configuration, timestamp_before, timestamp_after, initialization_in_accounting, remaining, self.core_server_url)
resource_type = session.query(ResourceType).filter_by(name = self.resource_type_name).one()
waiting_reservation = session.query(WaitingReservation).filter_by(reservation_id = reservation_id, resource_type_id = resource_type.id).first()
if waiting_reservation is None:
waiting_reservations = []
else:
#
# If it has not been assigned to any laboratory, then it might
# be waiting in the queue of that resource type (Waiting) or
# waiting for instances (WaitingInstances, meaning that there is
# no resource of that type implemented)
#
waiting_reservations = session.query(WaitingReservation)\
.filter(WaitingReservation.resource_type == waiting_reservation.resource_type).order_by(WaitingReservation.priority, WaitingReservation.id).all()
if waiting_reservation is None or waiting_reservation not in waiting_reservations:
#
# The position has changed and it is not in the list anymore!
# This has happened using WebLab Bot with 65 users.
#
return_current_status = True
else:
position = waiting_reservations.index(waiting_reservation)
remaining_working_instances = False
for resource_instance in waiting_reservation.resource_type.instances:
if resource_instance.slot is not None:
remaining_working_instances = True
break
finally:
session.close()
if return_current_status:
time.sleep(TIME_ANTI_RACE_CONDITIONS * random.random())
return self.get_reservation_status(reservation_id)
if remaining_working_instances:
return WSS.WaitingQueueStatus(reservation_id_with_route, position)
else:
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route, position)
################################################################
#
# Called when it is confirmed by the Laboratory Server.
#
@exc_checker
@logged()
@Override(Scheduler)
def confirm_experiment(self, reservation_id, lab_session_id, initial_configuration, exp_info):
self._remove_expired_reservations()
session = self.session_maker()
try:
if not self.reservations_manager.check(session, reservation_id):
return
possible_concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == reservation_id).first()
concrete_current_reservation = None
if possible_concrete_current_reservation is not None:
slot = possible_concrete_current_reservation.slot_reservation
if slot is not None:
current_resource_slot = slot.current_resource_slot
if current_resource_slot is not None:
resource_instance = current_resource_slot.resource_instance
if resource_instance is not None:
resource_type = resource_instance.resource_type
if resource_type is not None and resource_type.name == self.resource_type_name:
concrete_current_reservation = possible_concrete_current_reservation
if concrete_current_reservation is None:
return
concrete_current_reservation.lab_session_id = lab_session_id.id
concrete_current_reservation.initial_configuration = initial_configuration
concrete_current_reservation.exp_info = json.dumps(exp_info)
concrete_current_reservation.set_timestamp_after(self.time_provider.get_time())
session.commit()
finally:
session.close()
################################################################
#
# Called when the user disconnects or finishes the resource.
#
@exc_checker
@logged()
@Override(Scheduler)
def finish_reservation(self, reservation_id):
self._remove_expired_reservations()
session = self.session_maker()
try:
possible_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == reservation_id).first()
# Clean current reservation... if the current reservation is assigned to this scheduler
concrete_current_reservation = None
enqueue_free_experiment_args = None
if possible_current_reservation is not None:
slot = possible_current_reservation.slot_reservation
if slot is not None:
current_resource_slot = slot.current_resource_slot
if current_resource_slot is not None:
resource_instance = current_resource_slot.resource_instance
if resource_instance is not None:
resource_type = resource_instance.resource_type
if resource_type is not None and resource_type.name == self.resource_type_name:
concrete_current_reservation = possible_current_reservation
enqueue_free_experiment_args = self._clean_current_reservation(session, concrete_current_reservation)
db_resource_type = session.query(ResourceType).filter_by(name = self.resource_type_name).first()
reservation_to_delete = concrete_current_reservation or session.query(WaitingReservation).filter_by(reservation_id = reservation_id, resource_type = db_resource_type).first()
if reservation_to_delete is not None:
session.delete(reservation_to_delete)
session.commit()
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
finally:
session.close()
def _clean_current_reservation(self, session, concrete_current_reservation):
enqueue_free_experiment_args = None
if concrete_current_reservation is not None:
resource_instance = concrete_current_reservation.slot_reservation.current_resource_slot.resource_instance
if resource_instance is not None: # If the resource instance does not exist anymore, there is no need to call the free_experiment method
lab_session_id = concrete_current_reservation.lab_session_id
experiment_instance = None
for experiment_instance in resource_instance.experiment_instances:
if experiment_instance.experiment_type == concrete_current_reservation.current_reservation.reservation.experiment_type:
experiment_instance = experiment_instance
break
if experiment_instance is not None: # If the experiment instance doesn't exist, there is no need to call the free_experiment method
lab_coord_address = experiment_instance.laboratory_coord_address
reservation_id = concrete_current_reservation.current_reservation_id
enqueue_free_experiment_args = (lab_coord_address, reservation_id, lab_session_id, experiment_instance.to_experiment_instance_id())
self.reservations_manager.downgrade_confirmation(session, concrete_current_reservation.current_reservation_id)
return enqueue_free_experiment_args
def update(self):
self._update_queues()
#############################################################
#
# Take the queue of a given Resource Type and update it
#
@exc_checker
def _update_queues(self):
###########################################################
# There are reasons why a waiting reservation may not be
# able to be promoted while the next one is. For instance,
# if a user is waiting for "pld boards", but only for
# instances of "pld boards" which have a "ud-binary@Binary
# experiments" server running. If only a "ud-pld@PLD
# Experiments" is available, then this user will not be
# promoted and the another user which is waiting for a
# "ud-pld@PLD Experiments" can be promoted.
#
# Therefore, we have a list of the IDs of the waiting
# reservations we previously thought that they couldn't be
# promoted in this iteration. They will have another
# chance in the next run of _update_queues.
#
previously_waiting_reservation_ids = []
###########################################################
# While there are free instances and waiting reservations,
# take the first waiting reservation and set it to current
# reservation. Make this repeatedly because we want to
# commit each change
#
while True:
session = self.session_maker()
try:
resource_type = session.query(ResourceType).filter(ResourceType.name == self.resource_type_name).first()
#
# Retrieve the first waiting reservation. If there is no one that
# we haven't tried already, return
#
first_waiting_reservations = session.query(WaitingReservation).filter(WaitingReservation.resource_type == resource_type).order_by(WaitingReservation.priority, WaitingReservation.id)[:len(previously_waiting_reservation_ids) + 1]
first_waiting_reservation = None
for waiting_reservation in first_waiting_reservations:
if waiting_reservation.id not in previously_waiting_reservation_ids:
first_waiting_reservation = waiting_reservation
break
if first_waiting_reservation is None:
return # There is no waiting reservation for this resource that we haven't already tried
previously_waiting_reservation_ids.append(first_waiting_reservation.id)
#
# For the current resource_type, let's ask for
# all the resource instances available (i.e. those
# who have no SchedulingSchemaIndependentSlotReservation
# associated)
#
free_instances = session.query(CurrentResourceSlot)\
.select_from(join(CurrentResourceSlot, ResourceInstance))\
.filter(not_(CurrentResourceSlot.slot_reservations.any()))\
.filter(ResourceInstance.resource_type == resource_type)\
.order_by(CurrentResourceSlot.id).all()
if len(free_instances) == 0:
# If there is no free instance, just return
return
#
# Select the correct free_instance for the current student among
# all the free_instances
#
if self.randomize_instances:
randomized_free_instances = [ free_instance for free_instance in free_instances ]
random.shuffle(randomized_free_instances)
else:
randomized_free_instances = free_instances
for free_instance in randomized_free_instances:
resource_type = free_instance.resource_instance.resource_type
if resource_type is None:
continue # If suddenly the free_instance is not a free_instance anymore, try with other free_instance
#
# IMPORTANT: from here on every "continue" should first revoke the
# reservations_manager and resources_manager confirmations
#
self.reservations_manager.confirm(session, first_waiting_reservation.reservation_id)
slot_reservation = self.resources_manager.acquire_resource(session, free_instance)
total_time = first_waiting_reservation.time
initialization_in_accounting = first_waiting_reservation.initialization_in_accounting
start_time = self.time_provider.get_time()
concrete_current_reservation = ConcreteCurrentReservation(slot_reservation, first_waiting_reservation.reservation_id,
total_time, start_time, first_waiting_reservation.priority, first_waiting_reservation.initialization_in_accounting)
concrete_current_reservation.set_timestamp_before(self.time_provider.get_time())
client_initial_data = first_waiting_reservation.reservation.client_initial_data
request_info = json.loads(first_waiting_reservation.reservation.request_info)
username = request_info.get('username')
locale = request_info.get('locale')
reservation_id = first_waiting_reservation.reservation_id
if reservation_id is None:
break # If suddenly the waiting_reservation is not a waiting_reservation anymore, so reservation is None, go again to the while True.
requested_experiment_type = first_waiting_reservation.reservation.experiment_type
selected_experiment_instance = None
for experiment_instance in free_instance.resource_instance.experiment_instances:
if experiment_instance.experiment_type == requested_experiment_type:
selected_experiment_instance = experiment_instance
if selected_experiment_instance is None:
# This resource is not valid for this user, other free_instance should be
# selected. Try with other, but first clean the acquired resources
self.reservations_manager.downgrade_confirmation(session, first_waiting_reservation.reservation_id)
self.resources_manager.release_resource(session, slot_reservation)
continue
experiment_instance_id = ExperimentInstanceId(selected_experiment_instance.experiment_instance_id, requested_experiment_type.exp_name, requested_experiment_type.cat_name)
laboratory_coord_address = selected_experiment_instance.laboratory_coord_address
try:
session.delete(first_waiting_reservation)
session.add(concrete_current_reservation)
session.commit()
except IntegrityError as ie:
if DEBUG:
print("IntegrityError when adding concrete_current_reservation: ", sys.exc_info())
# Other scheduler confirmed the user or booked the reservation, rollback and try again
# But log just in case
log.log(
PriorityQueueScheduler, log.level.Warning,
"IntegrityError looping on update_queues: %s" % ie )
log.log_exc(PriorityQueueScheduler, log.level.Info)
session.rollback()
break
except Exception as e:
if DEBUG:
print("Other error when adding concrete_current_reservation: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"Exception looping on update_queues: %s" % e )
log.log_exc(PriorityQueueScheduler, log.level.Info)
session.rollback()
break
else:
#
# Enqueue the confirmation, since it might take a long time
# (for instance, if the laboratory server does not reply because
# of any network problem, or it just takes too much in replying),
# so this method might take too long. That's why we enqueue these
# petitions and run them in other threads.
#
deserialized_server_initial_data = {
'priority.queue.slot.length' : '%s' % total_time,
'priority.queue.slot.start' : '%s' % datetime.datetime.fromtimestamp(start_time),
'priority.queue.slot.initialization_in_accounting' : initialization_in_accounting,
'request.experiment_id.experiment_name' : experiment_instance_id.exp_name,
'request.experiment_id.category_name' : experiment_instance_id.cat_name,
'request.username' : username,
'request.full_name' : username,
'request.locale' : locale,
}
server_initial_data = json.dumps(deserialized_server_initial_data)
# server_initial_data will contain information such as "what was the last experiment used?".
# If a single resource was used by a binary experiment, then the next time may not require reprogramming the device
self.confirmer.enqueue_confirmation(laboratory_coord_address, reservation_id, experiment_instance_id, client_initial_data, server_initial_data, self.resource_type_name)
#
# After it, keep in the while True in order to add the next
# reservation
#
break
except (ConcurrentModificationError, IntegrityError) as ie:
# Something happened somewhere else, such as the user being confirmed twice, the experiment being reserved twice or so on.
# Rollback and start again
if DEBUG:
print("Other ConcurrentModificationError or IntegrityError in update_queues: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"Exception while updating queues, reverting and trying again: %s" % ie )
log.log_exc(PriorityQueueScheduler, log.level.Info)
session.rollback()
finally:
session.close()
################################################
#
# Remove all reservations whose session has expired
#
def _remove_expired_reservations(self):
session = self.session_maker()
try:
now = self.time_provider.get_time()
current_expiration_time = datetime.datetime.utcfromtimestamp(now - EXPIRATION_TIME)
reservations_removed = False
enqueue_free_experiment_args_retrieved = []
expired_query = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.expired_timestamp != 0).filter(ConcreteCurrentReservation.expired_timestamp < self.time_provider.get_time())
for expired_concrete_current_reservation in expired_query.all():
expired_reservation = expired_concrete_current_reservation.current_reservation_id
if expired_reservation is None:
continue # Maybe it's not an expired_reservation anymore
enqueue_free_experiment_args = self._clean_current_reservation(session, expired_concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(expired_concrete_current_reservation)
self.reservations_manager.delete(session, expired_reservation)
reservations_removed = True
for expired_reservation_id in self.reservations_manager.list_expired_reservations(session, current_expiration_time):
concrete_current_reservation = session.query(ConcreteCurrentReservation).filter(ConcreteCurrentReservation.current_reservation_id == expired_reservation_id).first()
if concrete_current_reservation is not None:
enqueue_free_experiment_args = self._clean_current_reservation(session, concrete_current_reservation)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
session.delete(concrete_current_reservation)
waiting_reservation = session.query(WaitingReservation).filter(WaitingReservation.reservation_id == expired_reservation_id).first()
if waiting_reservation is not None:
session.delete(waiting_reservation)
self.reservations_manager.delete(session, expired_reservation_id)
reservations_removed = True
if reservations_removed:
try:
session.commit()
except ConcurrentModificationError as e:
if DEBUG:
print("Other error when commiting when reservations_removed: ", sys.exc_info())
log.log(
PriorityQueueScheduler, log.level.Warning,
"IntegrityError: %s" % e )
log.log_exc(PriorityQueueScheduler, log.level.Info)
pass # Someone else removed these users before us.
else:
for enqueue_free_experiment_args in enqueue_free_experiment_args_retrieved:
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
else:
session.rollback()
finally:
session.close()
##############################################################
#
# ONLY FOR TESTING: It completely removes the whole database
#
@Override(Scheduler)
def _clean(self):
session = self.session_maker()
try:
for waiting_reservation in session.query(WaitingReservation).all():
session.delete(waiting_reservation)
for concrete_current_reservation in session.query(ConcreteCurrentReservation).all():
session.delete(concrete_current_reservation)
for current_resource_slot in session.query(CurrentResourceSlot).all():
session.delete(current_resource_slot)
session.commit()
except ConcurrentModificationError:
if DEBUG:
print("Error when cleaning: ", sys.exc_info())
pass # Another process is cleaning concurrently
finally:
session.close()
class PriorityQueueScheduler(Scheduler):
def __init__(self, generic_scheduler_arguments, randomize_instances = True, **kwargs):
super(PriorityQueueScheduler, self).__init__(generic_scheduler_arguments, **kwargs)
self.randomize_instances = randomize_instances
self._synchronizer = SchedulerTransactionsSynchronizer(self)
self._synchronizer.start()
@Override(Scheduler)
def stop(self):
self._synchronizer.stop()
@Override(Scheduler)
def is_remote(self):
return False
@exc_checker
@logged()
@Override(Scheduler)
@typecheck(typecheck.ANY, typecheck.ANY, Resource)
def removing_current_resource_slot(self, client, resource):
weblab_resource_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (resource.resource_type, resource.resource_instance)
current_reservation_ids = client.smembers(weblab_resource_instance_reservations)
if len(current_reservation_ids) > 0:
current_reservation_id = list(current_reservation_ids)[0]
if client.srem(weblab_resource_instance_reservations, current_reservation_id):
self.reservations_manager.downgrade_confirmation(current_reservation_id)
self.resources_manager.release_resource(resource)
# Remove data that was added when confirmed
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, current_reservation_id)
reservation_data_str = client.get(weblab_reservation_pqueue)
reservation_data = json.loads(reservation_data_str)
reservation_data.pop(ACTIVE_STATUS, None)
reservation_data.pop(TIMESTAMP_BEFORE, None)
reservation_data.pop(TIMESTAMP_AFTER, None)
reservation_data.pop(LAB_SESSION_ID, None)
reservation_data.pop(EXP_INFO, None)
reservation_data_str = json.dumps(reservation_data)
reservation_data = client.set(weblab_reservation_pqueue, reservation_data_str)
# Add back to the queue
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
filled_reservation_id = client.hget(weblab_resource_pqueue_map, current_reservation_id)
client.zadd(weblab_resource_pqueue_sorted, filled_reservation_id, -1)
return True
return False
@exc_checker
@logged()
@Override(Scheduler)
def reserve_experiment(self, reservation_id, experiment_id, time, priority, initialization_in_accounting, client_initial_data, request_info):
"""
priority: the less, the more priority
"""
client = self.redis_maker()
# For indexing purposes
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id)
weblab_resource_reservations = WEBLAB_RESOURCE_RESERVATIONS % self.resource_type_name
# Queue management
weblab_resource_pqueue_reservations = WEBLAB_RESOURCE_PQUEUE_RESERVATIONS % self.resource_type_name
weblab_resource_pqueue_positions = WEBLAB_RESOURCE_PQUEUE_POSITIONS % self.resource_type_name
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
# Within the same priority, we want all to sort all the requests by the order they came.
# In order to support this, we increase a long enough value and put it before the reservaiton_id
current_position = client.incr(weblab_resource_pqueue_positions)
filled_reservation_id = "%s_%s" % (str(current_position).zfill(100), reservation_id)
pipeline = client.pipeline()
pipeline.hset(weblab_resource_pqueue_map, reservation_id, filled_reservation_id)
pipeline.zadd(weblab_resource_pqueue_sorted, filled_reservation_id, priority)
pipeline.sadd(weblab_resource_reservations, reservation_id)
pipeline.sadd(weblab_resource_pqueue_reservations, reservation_id)
generic_data = {
TIME : time,
INITIALIZATION_IN_ACCOUNTING : initialization_in_accounting,
PRIORITY : priority,
}
pipeline.set(weblab_reservation_pqueue, json.dumps(generic_data))
pipeline.execute()
return self.get_reservation_status(reservation_id)
#######################################################################
#
# Given a reservation_id, it returns in which state the reservation is
#
@exc_checker
@logged()
@Override(Scheduler)
def get_reservation_status(self, reservation_id):
self._remove_expired_reservations()
expired = self.reservations_manager.update(reservation_id)
if expired:
self._delete_reservation(reservation_id)
raise ExpiredSessionError("Expired reservation")
self._synchronizer.request_and_wait()
reservation_id_with_route = '%s;%s.%s' % (reservation_id, reservation_id, self.core_server_route)
client = self.redis_maker()
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id)
reservation_data_str = client.get(weblab_reservation_pqueue)
if reservation_data_str is None:
log.log(
PriorityQueueScheduler, log.level.Error,
"get_reservation_status called with a reservation_id that is not registered (not found on weblab_reservation_pqueue). Returning a WaitingInstanceStatus")
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route, 50)
reservation_data = json.loads(reservation_data_str)
if ACTIVE_STATUS in reservation_data:
# Reserved or Waiting reservation
status = reservation_data[ACTIVE_STATUS]
# It may be just waiting for the experiment server to respond
if status == STATUS_WAITING_CONFIRMATION:
return WSS.WaitingConfirmationQueueStatus(reservation_id_with_route, self.core_server_url)
# Or the experiment server already responded and therefore we have all this data
str_lab_coord_address = reservation_data[LAB_COORD]
obtained_time = reservation_data[TIME]
initialization_in_accounting = reservation_data[INITIALIZATION_IN_ACCOUNTING]
lab_session_id = reservation_data[LAB_SESSION_ID]
initial_configuration = reservation_data[INITIAL_CONFIGURATION]
timestamp_before_tstamp = reservation_data[TIMESTAMP_BEFORE]
timestamp_after_tstamp = reservation_data[TIMESTAMP_AFTER]
if EXP_INFO in reservation_data and reservation_data[EXP_INFO]:
exp_info = json.loads(reservation_data[EXP_INFO])
else:
exp_info = {}
timestamp_before = datetime.datetime.fromtimestamp(timestamp_before_tstamp)
timestamp_after = datetime.datetime.fromtimestamp(timestamp_after_tstamp)
lab_coord_address = CoordAddress.translate(str_lab_coord_address)
if initialization_in_accounting:
before = timestamp_before_tstamp
else:
before = timestamp_after_tstamp
if before is not None:
remaining = (before + obtained_time) - self.time_provider.get_time()
else:
remaining = obtained_time
return WSS.LocalReservedStatus(reservation_id_with_route, lab_coord_address, SessionId.SessionId(lab_session_id), exp_info, obtained_time, initial_configuration, timestamp_before, timestamp_after, initialization_in_accounting, remaining, self.core_server_url)
# else it's waiting
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
filled_reservation_id = client.hget(weblab_resource_pqueue_map, reservation_id)
if filled_reservation_id is None:
log.log(
PriorityQueueScheduler, log.level.Error,
"get_reservation_status called with a reservation_id that is not registered (not found on the reservations map). Returning a WaitingInstanceStatus")
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route, 50)
position = client.zrank(weblab_resource_pqueue_sorted, filled_reservation_id)
if position is None: # It's not in the queue now
time.sleep(TIME_ANTI_RACE_CONDITIONS * random.random())
return self.get_reservation_status(reservation_id)
if self.resources_manager.are_resource_instances_working(self.resource_type_name):
return WSS.WaitingQueueStatus(reservation_id_with_route, position)
else:
return WSS.WaitingInstancesQueueStatus(reservation_id_with_route, position)
################################################################
#
# Called when it is confirmed by the Laboratory Server.
#
@exc_checker
@logged()
@Override(Scheduler)
def confirm_experiment(self, reservation_id, lab_session_id, initial_configuration, exp_info):
self._remove_expired_reservations()
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id)
client = self.redis_maker()
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
if pqueue_reservation_data_str is None:
return
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
resource_instance_str = pqueue_reservation_data.get(RESOURCE_INSTANCE)
if resource_instance_str is not None:
resource_instance = Resource.parse(resource_instance_str)
if not self.resources_manager.check_working(resource_instance):
# TODO: if the experiment is broken and the student is ACTIVE_STATUS, something should be done
#
return
pqueue_reservation_data[LAB_SESSION_ID] = lab_session_id.id
pqueue_reservation_data[INITIAL_CONFIGURATION] = initial_configuration
pqueue_reservation_data[TIMESTAMP_AFTER] = self.time_provider.get_time()
pqueue_reservation_data[ACTIVE_STATUS] = STATUS_RESERVED
pqueue_reservation_data[EXP_INFO] = json.dumps(exp_info)
pqueue_reservation_data_str = json.dumps(pqueue_reservation_data)
client.set(weblab_reservation_pqueue, pqueue_reservation_data_str)
################################################################
#
# Called when the user disconnects or finishes the resource.
#
@exc_checker
@logged()
@Override(Scheduler)
def finish_reservation(self, reservation_id):
self._remove_expired_reservations()
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id)
client = self.redis_maker()
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
if pqueue_reservation_data_str is None:
return
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
if ACTIVE_STATUS in pqueue_reservation_data:
enqueue_free_experiment_args = self._clean_current_reservation(reservation_id)
else:
enqueue_free_experiment_args = None
self._delete_reservation(reservation_id)
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
def _clean_current_reservation(self, reservation_id):
client = self.redis_maker()
enqueue_free_experiment_args = None
if reservation_id is not None:
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id)
reservation_data_str = client.get(weblab_reservation_pqueue)
if reservation_data_str is not None:
downgraded = self.reservations_manager.downgrade_confirmation(reservation_id)
if downgraded:
reservation_data = json.loads(reservation_data_str)
resource_instance_str = reservation_data.get(RESOURCE_INSTANCE)
if resource_instance_str is not None:
resource_instance = Resource.parse(resource_instance_str)
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (resource_instance.resource_type, resource_instance.resource_instance)
client.srem(weblab_resource_pqueue_instance_reservations, reservation_id)
# print "RELEASING AT _clean_current_reservation. SHOULD NEVER HAPPEN."
# self.resources_manager.release_resource(resource_instance)
lab_session_id = reservation_data.get(LAB_SESSION_ID)
experiment_instance_str = reservation_data.get(EXPERIMENT_INSTANCE)
experiment_instance_id = ExperimentInstanceId.parse(experiment_instance_str)
if experiment_instance_id is not None:
# If the experiment instance doesn't exist, there is no need to call the free_experiment method
lab_coord_address = reservation_data.get(LAB_COORD)
enqueue_free_experiment_args = (lab_coord_address, reservation_id, lab_session_id, experiment_instance_id)
# otherwise the student has been removed
return enqueue_free_experiment_args
def update(self):
self._update_queues()
#############################################################
#
# Take the queue of a given Resource Type and update it
#
@exc_checker
def _update_queues(self):
###########################################################
# There are reasons why a waiting reservation may not be
# able to be promoted while the next one is. For instance,
# if a user is waiting for "pld boards", but only for
# instances of "pld boards" which have a "ud-binary@Binary
# experiments" server running. If only a "ud-pld@PLD
# Experiments" is available, then this user will not be
# promoted and the another user which is waiting for a
# "ud-pld@PLD Experiments" can be promoted.
#
# Therefore, we have a list of the IDs of the waiting
# reservations we previously thought that they couldn't be
# promoted in this iteration. They will have another
# chance in the next run of _update_queues.
#
previously_waiting_reservation_ids = []
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
weblab_resource_slots = WEBLAB_RESOURCE_SLOTS % self.resource_type_name
###########################################################
# While there are free instances and waiting reservations,
# take the first waiting reservation and set it to current
# reservation. Make this repeatedly because we want to
# commit each change
#
while True:
client = self.redis_maker()
filled_waiting_reservation_ids = client.zrangebyscore(weblab_resource_pqueue_sorted, -10000, +10000, start=0, num=len(previously_waiting_reservation_ids) + 1)
first_waiting_reservation_id = None
for filled_waiting_reservation_id in filled_waiting_reservation_ids:
waiting_reservation_id = filled_waiting_reservation_id[filled_waiting_reservation_id.find('_')+1:]
if waiting_reservation_id not in previously_waiting_reservation_ids:
first_waiting_reservation_id = waiting_reservation_id
break
if first_waiting_reservation_id is None:
return # There is no waiting reservation for this resource that we haven't already tried
previously_waiting_reservation_ids.append(first_waiting_reservation_id)
#
# For the current resource_type, let's ask for
# all the resource instances available (i.e. those
# who are a member on weblab:resource:%s:slots )
#
free_instances = [ Resource(self.resource_type_name, resource_instance)
for resource_instance in client.smembers(weblab_resource_slots) ]
if len(free_instances) == 0:
# If there is no free instance, just return
return
#
# Select the correct free_instance for the current student among
# all the free_instances
#
if self.randomize_instances:
randomized_free_instances = [ free_instance for free_instance in free_instances ]
random.shuffle(randomized_free_instances)
else:
randomized_free_instances = sorted(free_instances, cmp=lambda r1, r2: cmp(r1.resource_type, r2.resource_type) or cmp(r1.resource_instance, r2.resource_instance))
for free_instance in randomized_free_instances:
#
# IMPORTANT: from here on every "continue" should first revoke the
# reservations_manager and resources_manager confirmations
#
working = self.resources_manager.check_working(free_instance)
if not working:
# The instance is not working
continue
confirmed = self.reservations_manager.confirm(first_waiting_reservation_id)
if not confirmed:
# student has already been confirmed somewhere else, so don't try with other
# instances, but rather with other student
break
acquired = self.resources_manager.acquire_resource(free_instance)
# print "ACQUIRED", free_instance, acquired, time.time()
if not acquired:
# the instance has been acquired by someone else. unconfirm student and
# try again with other free_instance
self.reservations_manager.downgrade_confirmation(first_waiting_reservation_id)
continue
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (self.resource_type_name, free_instance.resource_instance)
client.sadd(weblab_resource_pqueue_instance_reservations, first_waiting_reservation_id)
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, first_waiting_reservation_id)
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
reservation_data = self.reservations_manager.get_reservation_data(first_waiting_reservation_id)
if pqueue_reservation_data_str is None or reservation_data is None:
# the student is not here anymore; downgrading confirmation is not required
# but releasing the resource is; and skip the rest of the free instances
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations, first_waiting_reservation_id)
break
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
start_time = self.time_provider.get_time()
total_time = pqueue_reservation_data[TIME]
pqueue_reservation_data[START_TIME] = start_time
pqueue_reservation_data[TIMESTAMP_BEFORE] = start_time
pqueue_reservation_data[ACTIVE_STATUS] = STATUS_WAITING_CONFIRMATION
pqueue_reservation_data[RESOURCE_INSTANCE] = free_instance.to_weblab_str()
initialization_in_accounting = pqueue_reservation_data[INITIALIZATION_IN_ACCOUNTING]
client_initial_data = reservation_data[CLIENT_INITIAL_DATA]
request_info = json.loads(reservation_data[REQUEST_INFO])
username = request_info.get('username')
locale = request_info.get('locale')
requested_experiment_type = ExperimentId.parse(reservation_data[EXPERIMENT_TYPE])
selected_experiment_instance = None
experiment_instances = self.resources_manager.list_experiment_instance_ids_by_resource(free_instance)
for experiment_instance in experiment_instances:
if experiment_instance.to_experiment_id() == requested_experiment_type:
selected_experiment_instance = experiment_instance
if selected_experiment_instance is None:
# This resource is not valid for this user, other free_instance should be
# selected. Try with other, but first clean the acquired resources
self.reservations_manager.downgrade_confirmation(first_waiting_reservation_id)
self.resources_manager.release_resource(free_instance)
client.srem(weblab_resource_pqueue_instance_reservations, first_waiting_reservation_id)
continue
pqueue_reservation_data[EXPERIMENT_INSTANCE] = selected_experiment_instance.to_weblab_str()
laboratory_coord_address = self.resources_manager.get_laboratory_coordaddress_by_experiment_instance_id(selected_experiment_instance)
pqueue_reservation_data[LAB_COORD] = laboratory_coord_address
client.set(weblab_reservation_pqueue, json.dumps(pqueue_reservation_data))
filled_reservation_id = client.hget(weblab_resource_pqueue_map, first_waiting_reservation_id)
client.zrem(weblab_resource_pqueue_sorted, filled_reservation_id)
#
# Enqueue the confirmation, since it might take a long time
# (for instance, if the laboratory server does not reply because
# of any network problem, or it just takes too much in replying),
# so this method might take too long. That's why we enqueue these
# petitions and run them in other threads.
#
deserialized_server_initial_data = {
'priority.queue.slot.length' : '%s' % total_time,
'priority.queue.slot.start' : '%s' % datetime.datetime.fromtimestamp(start_time),
'priority.queue.slot.initialization_in_accounting' : initialization_in_accounting,
'request.experiment_id.experiment_name' : selected_experiment_instance.exp_name,
'request.experiment_id.category_name' : selected_experiment_instance.cat_name,
'request.username' : username,
'request.full_name' : username,
'request.locale' : locale,
# TODO: add the username and user full name here
}
server_initial_data = json.dumps(deserialized_server_initial_data)
# server_initial_data will contain information such as "what was the last experiment used?".
# If a single resource was used by a binary experiment, then the next time may not require reprogramming the device
self.confirmer.enqueue_confirmation(laboratory_coord_address, first_waiting_reservation_id, selected_experiment_instance, client_initial_data, server_initial_data, self.resource_type_name)
#
# After it, keep in the while True in order to add the next
# reservation
#
break
################################################
#
# Remove all reservations whose session has expired
#
@exc_checker
def _remove_expired_reservations(self):
now = self.time_provider.get_time()
enqueue_free_experiment_args_retrieved = []
client = self.redis_maker()
weblab_resource_pqueue_reservations = WEBLAB_RESOURCE_PQUEUE_RESERVATIONS % self.resource_type_name
reservations = [ reservation_id for reservation_id in client.smembers(weblab_resource_pqueue_reservations) ]
# Since there might be a lot of reservations, create a pipeline and retrieve
# every reservation data in a row
pipeline = client.pipeline()
for reservation_id in reservations:
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id)
pipeline.get(weblab_reservation_pqueue)
results = pipeline.execute()
for reservation_id, reservation_data in zip(reservations, results):
if reservation_data is not None:
data = json.loads(reservation_data)
if ACTIVE_STATUS in data:
total_time = data[TIME]
timestamp_before = data[TIMESTAMP_BEFORE]
timestamp_after = data.get(TIMESTAMP_AFTER)
initialization_in_accounting = data[INITIALIZATION_IN_ACCOUNTING]
# if timestamp_after is None and initialization should not be considered,
# then we can not calculate if the time has expired, so we skip it (it will
# be considered as expired for lack of LATEST_ACCESS
if timestamp_after is not None or initialization_in_accounting:
timestamp = timestamp_before if initialization_in_accounting else timestamp_after
if now >= timestamp + total_time: # Expired
enqueue_free_experiment_args = self._clean_current_reservation(reservation_id)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
self._delete_reservation(reservation_id)
self.reservations_manager.delete(reservation_id)
# Anybody with latest_access later than this point is expired
current_expiration_time = datetime.datetime.utcfromtimestamp(now - EXPIRATION_TIME)
for expired_reservation_id in self.reservations_manager.list_expired_reservations(current_expiration_time):
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, expired_reservation_id)
pqueue_reservation_data_str = client.get(weblab_reservation_pqueue)
if pqueue_reservation_data_str is None:
continue
pqueue_reservation_data = json.loads(pqueue_reservation_data_str)
if ACTIVE_STATUS in pqueue_reservation_data:
enqueue_free_experiment_args = self._clean_current_reservation(expired_reservation_id)
enqueue_free_experiment_args_retrieved.append(enqueue_free_experiment_args)
self._delete_reservation(expired_reservation_id)
self.reservations_manager.delete(expired_reservation_id)
for enqueue_free_experiment_args in enqueue_free_experiment_args_retrieved:
if enqueue_free_experiment_args is not None:
self.confirmer.enqueue_free_experiment(*enqueue_free_experiment_args)
def _delete_reservation(self, reservation_id):
weblab_resource_pqueue_reservations = WEBLAB_RESOURCE_PQUEUE_RESERVATIONS % self.resource_type_name
weblab_resource_pqueue_map = WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name
weblab_resource_pqueue_sorted = WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name
weblab_reservation_pqueue = WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id)
resource_instances = self.resources_manager.list_resource_instances_by_type(self.resource_type_name)
client = self.redis_maker()
pipeline = client.pipeline()
for resource_instance in resource_instances:
weblab_resource_pqueue_instance_reservations = WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (self.resource_type_name, resource_instance.resource_instance)
pipeline.srem(weblab_resource_pqueue_instance_reservations, reservation_id)
pipeline.srem(weblab_resource_pqueue_reservations, reservation_id)
pipeline.delete(weblab_reservation_pqueue)
pipeline.execute()
filled_reservation_id = client.hget(weblab_resource_pqueue_map, reservation_id)
client.hdel(weblab_resource_pqueue_map, reservation_id)
client.zrem(weblab_resource_pqueue_sorted, filled_reservation_id)
##############################################################
#
# ONLY FOR TESTING: It completely removes the whole database
#
@Override(Scheduler)
def _clean(self):
client = self.redis_maker()
for reservation_id in self.reservations_manager.list_all_reservations():
client.delete(WEBLAB_RESOURCE_RESERVATION_PQUEUE % (self.resource_type_name, reservation_id))
client.delete(WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (self.resource_type_name, '*'))
for resource_instance in self.resources_manager.list_resource_instances_by_type(self.resource_type_name):
client.delete(WEBLAB_RESOURCE_PQUEUE_INSTANCE_RESERVATIONS % (self.resource_type_name, resource_instance.resource_instance))
client.delete(WEBLAB_RESOURCE_PQUEUE_RESERVATIONS % self.resource_type_name)
client.delete(WEBLAB_RESOURCE_PQUEUE_POSITIONS % self.resource_type_name)
client.delete(WEBLAB_RESOURCE_PQUEUE_MAP % self.resource_type_name)
client.delete(WEBLAB_RESOURCE_PQUEUE_SORTED % self.resource_type_name)