def Cognitive(self, lease, nexttime):
lease.print_contents()
try:
self.__schedule_lease(lease, nexttime=nexttime)
self.logger.info("Cognitive lease lease #%i has been scheduled." %
lease.id)
lease.print_contents()
except NotSchedulableException, exc:
# Exception parameter set to save the node we tried to
self.logger.info(
"Cognitive lease lease request #%i cannot be scheduled now a time, it will be queued on node"
% (lease.id))
lease.start = Timestamp(Timestamp.UNSPECIFIED)
lease.preemptible = True
lease.cognitive = False
lease.set_state(Lease.STATE_QUEUED)
self.logger.info(
"Queued Cognitive lease request #%i, %i nodes for %s." %
(lease.id, lease.numnodes, lease.duration.requested))
get_persistence().persist_lease(lease)
try:
self.logger.info(
"Next request in the queue is lease %i. Attempting to schedule..."
% lease.id)
lease.print_contents()
self.__schedule_lease(lease, nexttime)
except NotSchedulableException, msg:
# Put back on queue: TO THE NEXT PROCESS
lease.queue_starttime = int(round(time.time() * 1000))
self.queue.enqueue(lease)
self.logger.info(
"Lease %i could not be scheduled at this time." % lease.id)
if get_config().get(
"backfilling") == constants.BACKFILLING_OFF:
done = True
def __preempt_lease(self, lease, preemption_time):
""" Preempts a lease.
This method preempts a lease such that any resources allocated
to that lease after a given time are freed up. This may require
scheduling the lease to suspend before that time, or cancelling
the lease altogether.
Arguments:
lease -- Lease to schedule.
preemption_time -- Time at which lease must be preempted
"""
self.logger.info("Preempting lease #%i..." % (lease.id))
self.logger.vdebug("Lease before preemption:")
lease.print_contents()
vmrr = lease.get_last_vmrr()
if vmrr.state == ResourceReservation.STATE_SCHEDULED and vmrr.start >= preemption_time:
self.logger.debug("Lease was set to start in the middle of the preempting lease.")
must_cancel_and_requeue = True
else:
susptype = get_config().get("suspension")
if susptype == constants.SUSPENSION_NONE:
must_cancel_and_requeue = True
else:
can_suspend = self.vm_scheduler.can_suspend_at(lease, preemption_time)
if not can_suspend:
self.logger.debug("Suspending the lease does not meet scheduling threshold.")
must_cancel_and_requeue = True
else:
if lease.numnodes > 1 and susptype == constants.SUSPENSION_SERIAL:
self.logger.debug("Can't suspend lease because only suspension of single-node leases is allowed.")
must_cancel_and_requeue = True
else:
self.logger.debug("Lease can be suspended")
must_cancel_and_requeue = False
if must_cancel_and_requeue:
self.logger.info("... lease #%i has been cancelled and requeued." % lease.id)
if lease.get_state() == Lease.STATE_SUSPENDED_SCHEDULED:
self.preparation_scheduler.cancel_preparation(lease, remove_files = False)
else:
self.preparation_scheduler.cancel_preparation(lease)
self.vm_scheduler.cancel_vm(vmrr)
lease.remove_vmrr(vmrr)
# TODO: Take into account other states
if lease.get_state() == Lease.STATE_SUSPENDED_SCHEDULED:
lease.set_state(Lease.STATE_SUSPENDED_QUEUED)
else:
lease.set_state(Lease.STATE_QUEUED)
self.__enqueue_in_order(lease)
else:
self.logger.info("... lease #%i will be suspended at %s." % (lease.id, preemption_time))
self.vm_scheduler.preempt_vm(vmrr, preemption_time)
get_persistence().persist_lease(lease)
self.logger.vdebug("Lease after preemption:")
lease.print_contents()
def notify_event(self, lease, event):
"""Notifies an event that affects a lease.
This is the entry point of asynchronous events into the scheduler. Currently,
the only supported event is the premature end of a VM (i.e., before its
scheduled end). Other events will emerge when we integrate Haizea with OpenNebula 1.4,
since that version will support sending asynchronous events to Haizea.
Arguments:
lease -- Lease the event refers to
event -- Event type
"""
time = get_clock().get_time()
if event == constants.EVENT_END_VM:
vmrr = lease.get_last_vmrr()
self._handle_end_rr(vmrr)
# TODO: Exception handling
self.vm_scheduler._handle_unscheduled_end_vm(lease, vmrr)
self._handle_end_lease(lease)
get_persistence().persist_lease(lease)
# We need to reevaluate the schedule to see if there are any
# leases scheduled in the future that could be rescheduled
# to start earlier
nexttime = get_clock().get_next_schedulable_time()
self.reevaluate_schedule(nexttime)
def notify_event(self, lease, event):
"""Notifies an event that affects a lease.
This is the entry point of asynchronous events into the scheduler. Currently,
the only supported event is the premature end of a VM (i.e., before its
scheduled end). Other events will emerge when we integrate Haizea with OpenNebula 1.4,
since that version will support sending asynchronous events to Haizea.
Arguments:
lease -- Lease the event refers to
event -- Event type
"""
time = get_clock().get_time()
if event == constants.EVENT_END_VM:
vmrr = lease.get_last_vmrr()
self._handle_end_rr(vmrr)
# TODO: Exception handling
self.vm_scheduler._handle_unscheduled_end_vm(lease, vmrr)
self._handle_end_lease(lease)
get_persistence().persist_lease(lease)
# We need to reevaluate the schedule to see if there are any
# leases scheduled in the future that could be rescheduled
# to start earlier
nexttime = get_clock().get_next_schedulable_time()
self.reevaluate_schedule(nexttime)
def Cognitive(self,lease, nexttime):
lease.print_contents()
try:
self.__schedule_lease(lease, nexttime=nexttime)
self.logger.info("Cognitive lease lease #%i has been scheduled." % lease.id)
lease.print_contents()
except NotSchedulableException, exc:
# Exception parameter set to save the node we tried to
self.logger.info("Cognitive lease lease request #%i cannot be scheduled now a time, it will be queued on node" % (lease.id))
lease.start = Timestamp(Timestamp.UNSPECIFIED)
lease.preemptible = True
lease.cognitive = False
lease.set_state(Lease.STATE_QUEUED)
self.logger.info("Queued Cognitive lease request #%i, %i nodes for %s." % (lease.id, lease.numnodes, lease.duration.requested))
get_persistence().persist_lease(lease)
try:
self.logger.info("Next request in the queue is lease %i. Attempting to schedule..." % lease.id)
lease.print_contents()
self.__schedule_lease(lease, nexttime)
except NotSchedulableException, msg:
# Put back on queue: TO THE NEXT PROCESS
lease.queue_starttime = int(round(time.time() * 1000))
self.queue.enqueue(lease)
self.logger.info("Lease %i could not be scheduled at this time." % lease.id)
if get_config().get("backfilling") == constants.BACKFILLING_OFF:
done = True
def schedule(self, nexttime):
""" The main scheduling function
The scheduling function looks at all pending requests and schedules them.
Note that most of the actual scheduling code is contained in the
__schedule_lease method and in the VMScheduler and PreparationScheduler classes.
Arguments:
nexttime -- The next time at which the scheduler can allocate resources.
"""
# Get pending leases
pending_leases = self.leases.get_leases_by_state(Lease.STATE_PENDING)
ar_leases = [req for req in pending_leases if req.get_type() == Lease.ADVANCE_RESERVATION]
im_leases = [req for req in pending_leases if req.get_type() == Lease.IMMEDIATE]
be_leases = [req for req in pending_leases if req.get_type() == Lease.BEST_EFFORT]
cognitive_leases = [req for req in pending_leases if req.get_type() == Lease.COGNITIVE]
#############################################################################################################################
#############################################################################################################################
# Second change (1): process cognitive leases directly #
#############################################################################################################################
#############################################################################################################################
for lease in cognitive_leases:
thread = Thread(target=self.Cognitive, args=(lease,nexttime) )
thread.start()
thread.join()
#############################################################################################################################
#############################################################################################################################
# Queue best-effort leases
for lease in be_leases:
self.__enqueue(lease)
lease.set_state(Lease.STATE_QUEUED)
self.logger.info("Queued best-effort lease request #%i, %i nodes for %s." % (lease.id, lease.numnodes, lease.duration.requested))
get_persistence().persist_lease(lease)
#Schedule immediate leases
for lease in im_leases:
self.logger.info("Scheduling immediate lease #%i (%i nodes)" % (lease.id, lease.numnodes))
lease.print_contents()
try:
self.__schedule_lease(lease, nexttime=nexttime)
self.logger.info("Immediate lease #%i has been scheduled." % lease.id)
lease.print_contents()
except NotSchedulableException, exc:
self.logger.info("Immediate lease request #%i cannot be scheduled: %s" % (lease.id, exc.reason))
lease.set_state(Lease.STATE_REJECTED)
self.completed_leases.add(lease)
self.accounting.at_lease_done(lease)
self.leases.remove(lease)
get_persistence().persist_lease(lease)
def process_ending_reservations(self, nowtime):
"""Processes ending reservations
This method checks the slottable to see if there are any reservations that are
ending at "nowtime". If so, the appropriate handler is called.
Arguments:
nowtime -- Time at which to check for starting/ending reservations.
"""
# Find starting/ending reservations
ending = self.slottable.get_reservations_ending_at(nowtime)
ending = [
res for res in ending
if res.state == ResourceReservation.STATE_ACTIVE
]
# Process ending reservations
for rr in ending:
lease = rr.lease
self._handle_end_rr(rr)
# Call the appropriate handler, and catch exceptions and errors.
try:
self.handlers[type(rr)].on_end(lease, rr)
# A RescheduleLeaseException indicates that the lease has to be rescheduled
except RescheduleLeaseException, exc:
# Currently, the only leases that get rescheduled are best-effort leases,
# once they've been suspended.
if rr.lease.get_type() == Lease.BEST_EFFORT:
if lease.get_state() == Lease.STATE_SUSPENDED_PENDING:
# Put back in the queue, in the same order it arrived
self.__enqueue_in_order(lease)
lease.set_state(Lease.STATE_SUSPENDED_QUEUED)
get_persistence().persist_queue(self.queue)
else:
raise InconsistentLeaseStateError(
lease, doing="rescheduling best-effort lease")
# A NormalEndLeaseException indicates that the end of this reservations marks
# the normal end of the lease.
except NormalEndLeaseException, msg:
self._handle_end_lease(lease)
def process_ending_reservations(self, nowtime):
"""Processes ending reservations
This method checks the slottable to see if there are any reservations that are
ending at "nowtime". If so, the appropriate handler is called.
Arguments:
nowtime -- Time at which to check for starting/ending reservations.
"""
# Find starting/ending reservations
ending = self.slottable.get_reservations_ending_at(nowtime)
ending = [res for res in ending if res.state == ResourceReservation.STATE_ACTIVE]
# Process ending reservations
for rr in ending:
lease = rr.lease
self._handle_end_rr(rr)
# Call the appropriate handler, and catch exceptions and errors.
try:
self.handlers[type(rr)].on_end(lease, rr)
# A RescheduleLeaseException indicates that the lease has to be rescheduled
except RescheduleLeaseException, exc:
# Currently, the only leases that get rescheduled are best-effort leases,
# once they've been suspended.
if rr.lease.get_type() == Lease.BEST_EFFORT:
if lease.get_state() == Lease.STATE_SUSPENDED_PENDING:
# Put back in the queue, in the same order it arrived
self.__enqueue_in_order(lease)
lease.set_state(Lease.STATE_SUSPENDED_QUEUED)
get_persistence().persist_queue(self.queue)
else:
raise InconsistentLeaseStateError(lease, doing = "rescheduling best-effort lease")
# A NormalEndLeaseException indicates that the end of this reservations marks
# the normal end of the lease.
except NormalEndLeaseException, msg:
self._handle_end_lease(lease)
def fail_lease(self, lease, exc=None):
"""Transitions a lease to a failed state, and does any necessary cleaning up
Arguments:
lease -- Lease to fail
exc -- The exception that made the lease fail
"""
treatment = get_config().get("lease-failure-handling")
if treatment == constants.ONFAILURE_CANCEL:
# In this case, a lease failure is handled by cancelling the lease,
# but allowing Haizea to continue to run normally.
rrs = lease.get_scheduled_reservations()
for r in rrs:
self.slottable.remove_reservation(r)
lease.set_state(Lease.STATE_FAIL)
self.completed_leases.add(lease)
self.leases.remove(lease)
get_persistence().persist_lease(lease)
elif treatment == constants.ONFAILURE_EXIT or treatment == constants.ONFAILURE_EXIT_RAISE:
# In this case, a lease failure makes Haizea exit. This is useful when debugging,
# so we can immediately know about any errors.
raise UnrecoverableError(exc)
def fail_lease(self, lease, exc=None):
"""Transitions a lease to a failed state, and does any necessary cleaning up
Arguments:
lease -- Lease to fail
exc -- The exception that made the lease fail
"""
treatment = get_config().get("lease-failure-handling")
if treatment == constants.ONFAILURE_CANCEL:
# In this case, a lease failure is handled by cancelling the lease,
# but allowing Haizea to continue to run normally.
rrs = lease.get_scheduled_reservations()
for r in rrs:
self.slottable.remove_reservation(r)
lease.set_state(Lease.STATE_FAIL)
self.completed_leases.add(lease)
self.leases.remove(lease)
get_persistence().persist_lease(lease)
elif treatment == constants.ONFAILURE_EXIT or treatment == constants.ONFAILURE_EXIT_RAISE:
# In this case, a lease failure makes Haizea exit. This is useful when debugging,
# so we can immediately know about any errors.
raise UnrecoverableError(exc)
def request_lease(self, lease):
"""Requests a leases. This is the entry point of leases into the scheduler.
Request a lease. The decision on whether to accept or reject a
lease is deferred to the policy manager (through its admission
control policy).
If the policy determines the lease can be
accepted, it is marked as "Pending". This still doesn't
guarantee that the lease will be scheduled (e.g., an AR lease
could still be rejected if the scheduler determines there are no
resources for it; but that is a *scheduling* decision, not a admission
control policy decision). The ultimate fate of the lease is determined
the next time the scheduling function is called.
If the policy determines the lease cannot be accepted, it is marked
as rejected.
Arguments:
lease -- Lease object. Its state must be STATE_NEW.
"""
self.logger.info("Lease #%i has been requested." % lease.id)
if lease.submit_time == None:
lease.submit_time = round_datetime(get_clock().get_time())
lease.print_contents()
lease.set_state(Lease.STATE_PENDING)
if get_policy().accept_lease(lease):
self.logger.info("Lease #%i has been marked as pending." %
lease.id)
self.leases.add(lease)
else:
self.logger.info("Lease #%i has not been accepted" % lease.id)
lease.set_state(Lease.STATE_REJECTED)
self.completed_leases.add(lease)
self.accounting.at_lease_request(lease)
get_persistence().persist_lease(lease)
def request_lease(self, lease):
"""Requests a leases. This is the entry point of leases into the scheduler.
Request a lease. The decision on whether to accept or reject a
lease is deferred to the policy manager (through its admission
control policy).
If the policy determines the lease can be
accepted, it is marked as "Pending". This still doesn't
guarantee that the lease will be scheduled (e.g., an AR lease
could still be rejected if the scheduler determines there are no
resources for it; but that is a *scheduling* decision, not a admission
control policy decision). The ultimate fate of the lease is determined
the next time the scheduling function is called.
If the policy determines the lease cannot be accepted, it is marked
as rejected.
Arguments:
lease -- Lease object. Its state must be STATE_NEW.
"""
self.logger.info("Lease #%i has been requested." % lease.id)
if lease.submit_time == None:
lease.submit_time = round_datetime(get_clock().get_time())
lease.print_contents()
lease.set_state(Lease.STATE_PENDING)
if get_policy().accept_lease(lease):
self.logger.info("Lease #%i has been marked as pending." % lease.id)
self.leases.add(lease)
else:
self.logger.info("Lease #%i has not been accepted" % lease.id)
lease.set_state(Lease.STATE_REJECTED)
self.completed_leases.add(lease)
self.accounting.at_lease_request(lease)
get_persistence().persist_lease(lease)
def __preempt_lease(self, lease, preemption_time):
""" Preempts a lease.
This method preempts a lease such that any resources allocated
to that lease after a given time are freed up. This may require
scheduling the lease to suspend before that time, or cancelling
the lease altogether.
Arguments:
lease -- Lease to schedule.
preemption_time -- Time at which lease must be preempted
"""
self.logger.info("Preempting lease #%i..." % (lease.id))
self.logger.vdebug("Lease before preemption:")
lease.print_contents()
vmrr = lease.get_last_vmrr()
if vmrr.state == ResourceReservation.STATE_SCHEDULED and vmrr.start >= preemption_time:
self.logger.debug(
"Lease was set to start in the middle of the preempting lease."
)
must_cancel_and_requeue = True
else:
susptype = get_config().get("suspension")
if susptype == constants.SUSPENSION_NONE:
must_cancel_and_requeue = True
else:
can_suspend = self.vm_scheduler.can_suspend_at(
lease, preemption_time)
if not can_suspend:
self.logger.debug(
"Suspending the lease does not meet scheduling threshold."
)
must_cancel_and_requeue = True
else:
if lease.numnodes > 1 and susptype == constants.SUSPENSION_SERIAL:
self.logger.debug(
"Can't suspend lease because only suspension of single-node leases is allowed."
)
must_cancel_and_requeue = True
else:
self.logger.debug("Lease can be suspended")
must_cancel_and_requeue = False
if must_cancel_and_requeue:
self.logger.info("... lease #%i has been cancelled and requeued." %
lease.id)
self.preparation_scheduler.cancel_preparation(lease)
self.vm_scheduler.cancel_vm(vmrr)
lease.remove_vmrr(vmrr)
# TODO: Take into account other states
if lease.get_state() == Lease.STATE_SUSPENDED_SCHEDULED:
lease.set_state(Lease.STATE_SUSPENDED_QUEUED)
else:
lease.set_state(Lease.STATE_QUEUED)
self.__enqueue_in_order(lease)
else:
self.logger.info("... lease #%i will be suspended at %s." %
(lease.id, preemption_time))
self.vm_scheduler.preempt_vm(vmrr, preemption_time)
get_persistence().persist_lease(lease)
self.logger.vdebug("Lease after preemption:")
lease.print_contents()
def schedule(self, nexttime):
""" The main scheduling function
The scheduling function looks at all pending requests and schedules them.
Note that most of the actual scheduling code is contained in the
__schedule_lease method and in the VMScheduler and PreparationScheduler classes.
Arguments:
nexttime -- The next time at which the scheduler can allocate resources.
"""
# Get pending leases
pending_leases = self.leases.get_leases_by_state(Lease.STATE_PENDING)
# we process all the leases at one time, without notion of queued leases
# and now leases
for choice in generate_combination(pending_leases):
# we first check if IM leases are feasible
im_leases = [l for l in choice if l.get_type() == Lease.IMMEDIATE]
# Process leases that have to be queued. Right now, only best-effort leases get queued.
queue_leases = [req for req in pending_leases if req.get_type() == Lease.BEST_EFFORT]
# Queue leases
for lease in queue_leases:
self.__enqueue(lease)
lease.set_state(Lease.STATE_QUEUED)
self.logger.info("Queued lease request #%i, %i nodes for %s." % (lease.id, lease.numnodes, lease.duration.requested))
get_persistence().persist_lease(lease)
# Process leases that have to be scheduled right away. Right now, this is any
# lease that is not a best-effort lease (ARs, immediate, and deadlined leases)
now_leases = [req for req in pending_leases if req.get_type() != Lease.BEST_EFFORT]
# Schedule leases
for lease in now_leases:
lease_type = Lease.type_str[lease.get_type()]
self.logger.info("Scheduling lease #%i (%i nodes) -- %s" % (lease.id, lease.numnodes, lease_type))
if lease.get_type() == Lease.ADVANCE_RESERVATION:
self.logger.info("From %s to %s" % (lease.start.requested, lease.start.requested + lease.duration.requested))
elif lease.get_type() == Lease.DEADLINE:
self.logger.info("Starting at %s. Deadline: %s" % (lease.start.requested, lease.deadline))
lease.print_contents()
try:
self.__schedule_lease(lease, nexttime=nexttime)
self.logger.info("Lease #%i has been scheduled." % lease.id)
## BEGIN NOT-FIT-FOR-PRODUCTION CODE
## This should happen when the lease is requested.
get_policy().pricing.feedback(lease)
## END NOT-FIT-FOR-PRODUCTION CODE
lease.print_contents()
except NotSchedulableException, exc:
self.logger.info("Lease request #%i cannot be scheduled: %s" % (lease.id, exc.reason))
## BEGIN NOT-FIT-FOR-PRODUCTION CODE
## This should happen when the lease is requested.
if lease.price == -1:
lease.set_state(Lease.STATE_REJECTED_BY_USER)
else:
lease.set_state(Lease.STATE_REJECTED)
self.completed_leases.add(lease)
self.accounting.at_lease_done(lease)
## BEGIN NOT-FIT-FOR-PRODUCTION CODE
## This should happen when the lease is requested.
get_policy().pricing.feedback(lease)
## END NOT-FIT-FOR-PRODUCTION CODE
self.leases.remove(lease)
get_persistence().persist_lease(lease)
def __schedule_lease(self, lease, nexttime):
""" Schedules a lease.
This method orchestrates the preparation and VM scheduler to
schedule a lease.
Arguments:
lease -- Lease to schedule.
nexttime -- The next time at which the scheduler can allocate resources.
"""
lease_state = lease.get_state()
migration = get_config().get("migration")
# Determine earliest start time in each node
if lease_state == Lease.STATE_PENDING or lease_state == Lease.STATE_QUEUED:
# This lease might require preparation. Ask the preparation
# scheduler for the earliest starting time.
earliest = self.preparation_scheduler.find_earliest_starting_times(
lease, nexttime)
elif lease_state == Lease.STATE_SUSPENDED_PENDING or lease_state == Lease.STATE_SUSPENDED_QUEUED:
# This lease may have to be migrated.
# We have to ask both the preparation scheduler and the VM
# scheduler what would be the earliest possible starting time
# on each node, assuming we have to transfer files between
# nodes.
node_ids = self.slottable.nodes.keys()
earliest = {}
if migration == constants.MIGRATE_NO:
# If migration is disabled, the earliest starting time
# is simply nexttime.
for node in node_ids:
earliest[node] = EarliestStartingTime(
nexttime, EarliestStartingTime.EARLIEST_NOPREPARATION)
else:
# Otherwise, we ask the preparation scheduler and the VM
# scheduler how long it would take them to migrate the
# lease state.
prep_migr_time = self.preparation_scheduler.estimate_migration_time(
lease)
vm_migr_time = self.vm_scheduler.estimate_migration_time(lease)
for node in node_ids:
earliest[node] = EarliestStartingTime(
nexttime + prep_migr_time + vm_migr_time,
EarliestStartingTime.EARLIEST_MIGRATION)
else:
raise InconsistentLeaseStateError(
lease, doing="scheduling a best-effort lease")
# Now, we give the lease to the VM scheduler, along with the
# earliest possible starting times. If the VM scheduler can
# schedule VMs for this lease, it will return a resource reservation
# that we can add to the slot table, along with a list of
# leases that have to be preempted.
# If the VM scheduler can't schedule the VMs, it will throw an
# exception (we don't catch it here, and it is just thrown up
# to the calling method.
(vmrr,
preemptions) = self.vm_scheduler.schedule(lease, nexttime, earliest)
if vmrr == None and preemptions == None:
return
# If scheduling the lease involves preempting other leases,
# go ahead and preempt them.
if len(preemptions) > 0:
self.logger.info(
"Must preempt leases %s to make room for lease #%i" %
([l.id for l in preemptions], lease.id))
for l in preemptions:
self.__preempt_lease(l, preemption_time=vmrr.start)
l.strv_counter = l.strv_counter + 1
# Schedule lease preparation
is_ready = False
preparation_rrs = []
if lease_state in (Lease.STATE_SUSPENDED_PENDING,
Lease.STATE_SUSPENDED_QUEUED
) and migration != constants.MIGRATE_NO:
# The lease might require migration
migr_rrs = self.preparation_scheduler.schedule_migration(
lease, vmrr, nexttime)
if len(migr_rrs) > 0:
end_migr = migr_rrs[-1].end
else:
end_migr = nexttime
migr_rrs += self.vm_scheduler.schedule_migration(
lease, vmrr, end_migr)
migr_rrs.reverse()
for migr_rr in migr_rrs:
vmrr.pre_rrs.insert(0, migr_rr)
if len(migr_rrs) == 0:
is_ready = True
elif lease_state in (Lease.STATE_SUSPENDED_PENDING,
Lease.STATE_SUSPENDED_QUEUED
) and migration == constants.MIGRATE_NO:
# No migration means the lease is ready
is_ready = True
elif lease_state in (Lease.STATE_PENDING, Lease.STATE_QUEUED):
# The lease might require initial preparation
preparation_rrs, is_ready = self.preparation_scheduler.schedule(
lease, vmrr, earliest)
# At this point, the lease is feasible.
# Commit changes by adding RRs to lease and to slot table
# Add preparation RRs (if any) to lease
for rr in preparation_rrs:
lease.append_preparationrr(rr)
# Add VMRR to lease
lease.append_vmrr(vmrr)
# Add resource reservations to slottable
# Preparation RRs (if any)
for rr in preparation_rrs:
self.slottable.add_reservation(rr)
# Pre-VM RRs (if any)
for rr in vmrr.pre_rrs:
self.slottable.add_reservation(rr)
# VM
self.slottable.add_reservation(vmrr)
# Post-VM RRs (if any)
for rr in vmrr.post_rrs:
self.slottable.add_reservation(rr)
# Change lease state
if lease_state == Lease.STATE_PENDING or lease_state == Lease.STATE_QUEUED:
lease.set_state(Lease.STATE_SCHEDULED)
if is_ready:
lease.set_state(Lease.STATE_READY)
elif lease_state == Lease.STATE_SUSPENDED_PENDING or lease_state == Lease.STATE_SUSPENDED_QUEUED:
lease.set_state(Lease.STATE_SUSPENDED_SCHEDULED)
get_persistence().persist_lease(lease)
lease.print_contents()
def __schedule_lease(self, lease, nexttime):
""" Schedules a lease.
This method orchestrates the preparation and VM scheduler to
schedule a lease.
Arguments:
lease -- Lease to schedule.
nexttime -- The next time at which the scheduler can allocate resources.
"""
lease_state = lease.get_state()
migration = get_config().get("migration")
# Determine earliest start time in each node
if lease_state == Lease.STATE_PENDING or lease_state == Lease.STATE_QUEUED:
# This lease might require preparation. Ask the preparation
# scheduler for the earliest starting time.
earliest = self.preparation_scheduler.find_earliest_starting_times(lease, nexttime)
elif lease_state == Lease.STATE_SUSPENDED_PENDING or lease_state == Lease.STATE_SUSPENDED_QUEUED:
# This lease may have to be migrated.
# We have to ask both the preparation scheduler and the VM
# scheduler what would be the earliest possible starting time
# on each node, assuming we have to transfer files between
# nodes.
node_ids = self.slottable.nodes.keys()
earliest = {}
if migration == constants.MIGRATE_NO:
# If migration is disabled, the earliest starting time
# is simply nexttime.
for node in node_ids:
earliest[node] = EarliestStartingTime(nexttime, EarliestStartingTime.EARLIEST_NOPREPARATION)
else:
# Otherwise, we ask the preparation scheduler and the VM
# scheduler how long it would take them to migrate the
# lease state.
prep_migr_time = self.preparation_scheduler.estimate_migration_time(lease)
vm_migr_time = self.vm_scheduler.estimate_migration_time(lease)
for node in node_ids:
earliest[node] = EarliestStartingTime(nexttime + prep_migr_time + vm_migr_time, EarliestStartingTime.EARLIEST_MIGRATION)
else:
raise InconsistentLeaseStateError(lease, doing = "scheduling a best-effort lease")
# Now, we give the lease to the VM scheduler, along with the
# earliest possible starting times. If the VM scheduler can
# schedule VMs for this lease, it will return a resource reservation
# that we can add to the slot table, along with a list of
# leases that have to be preempted.
# If the VM scheduler can't schedule the VMs, it will throw an
# exception (we don't catch it here, and it is just thrown up
# to the calling method.
(vmrr, preemptions) = self.vm_scheduler.schedule(lease, lease.duration.get_remaining_duration(), nexttime, earliest)
## BEGIN NOT-FIT-FOR-PRODUCTION CODE
## Pricing shouldn't live here. Instead, it should happen before a lease is accepted
## It is being done here in the interest of developing a first prototype
## that incorporates pricing in simulations (but not interactively yet)
# Call pricing policy
lease_price = get_policy().price_lease(lease, preemptions)
# Determine whether to accept price or not (this in particular
# should happen in the lease admission step)
if lease.extras.has_key("simul_userrate"):
user_rate = float(lease.extras["simul_userrate"])
if get_config().get("policy.pricing") != "free":
user_price = get_policy().pricing.get_base_price(lease, user_rate)
# We want to record the rate at which the lease was priced
lease.extras["rate"] = get_policy().pricing.rate
if lease_price > user_price:
lease.price = -1
lease.extras["rejected_price"] = lease_price
raise NotSchedulableException, "Lease priced at %.2f. User is only willing to pay %.2f" % (lease_price, user_price)
lease.price = lease_price
## END NOT-FIT-FOR-PRODUCTION CODE
# Schedule lease preparation
is_ready = False
preparation_rrs = []
if lease_state in (Lease.STATE_SUSPENDED_PENDING, Lease.STATE_SUSPENDED_QUEUED) and migration != constants.MIGRATE_NO:
# The lease might require migration
migr_rrs = self.preparation_scheduler.schedule_migration(lease, vmrr, nexttime)
if len(migr_rrs) > 0:
end_migr = migr_rrs[-1].end
else:
end_migr = nexttime
migr_rrs += self.vm_scheduler.schedule_migration(lease, vmrr, end_migr)
migr_rrs.reverse()
for migr_rr in migr_rrs:
vmrr.pre_rrs.insert(0, migr_rr)
if len(migr_rrs) == 0:
is_ready = True
elif lease_state in (Lease.STATE_SUSPENDED_PENDING, Lease.STATE_SUSPENDED_QUEUED) and migration == constants.MIGRATE_NO:
# No migration means the lease is ready
is_ready = True
elif lease_state in (Lease.STATE_PENDING, Lease.STATE_QUEUED):
# The lease might require initial preparation
preparation_rrs, is_ready = self.preparation_scheduler.schedule(lease, vmrr, earliest, nexttime)
# If scheduling the lease involves preempting other leases,
# go ahead and preempt them.
if len(preemptions) > 0:
self.logger.info("Must preempt leases %s to make room for lease #%i" % ([l.id for l in preemptions], lease.id))
for l in preemptions:
self.__preempt_lease(l, preemption_time=vmrr.start)
# At this point, the lease is feasible.
# Commit changes by adding RRs to lease and to slot table
# Add preparation RRs (if any) to lease
for rr in preparation_rrs:
lease.append_preparationrr(rr)
# Add VMRR to lease
lease.append_vmrr(vmrr)
# Add resource reservations to slottable
# Preparation RRs (if any)
for rr in preparation_rrs:
self.slottable.add_reservation(rr)
# Pre-VM RRs (if any)
for rr in vmrr.pre_rrs:
self.slottable.add_reservation(rr)
# VM
self.slottable.add_reservation(vmrr)
# Post-VM RRs (if any)
for rr in vmrr.post_rrs:
self.slottable.add_reservation(rr)
# Change lease state
if lease_state == Lease.STATE_PENDING or lease_state == Lease.STATE_QUEUED:
lease.set_state(Lease.STATE_SCHEDULED)
if is_ready:
lease.set_state(Lease.STATE_READY)
elif lease_state == Lease.STATE_SUSPENDED_PENDING or lease_state == Lease.STATE_SUSPENDED_QUEUED:
lease.set_state(Lease.STATE_SUSPENDED_SCHEDULED)
get_persistence().persist_lease(lease)
lease.print_contents()
try:
self.__schedule_lease(lease, nexttime)
self.logger.info("AR lease #%i has been scheduled." % lease.id)
lease.print_contents()
except NotSchedulableException, exc:
self.logger.info("AR lease request #%i cannot be scheduled: %s" % (lease.id, exc.reason))
lease.set_state(Lease.STATE_REJECTED)
self.completed_leases.add(lease)
self.accounting.at_lease_done(lease)
self.leases.remove(lease)
get_persistence().persist_lease(lease)
# Process queue (i.e., traverse queue in search of leases that can be scheduled)
self.__process_queue(nexttime)
get_persistence().persist_queue(self.queue)
def process_starting_reservations(self, nowtime):
"""Processes starting reservations
This method checks the slottable to see if there are any reservations that are
starting at "nowtime". If so, the appropriate handler is called.
Arguments:
nowtime -- Time at which to check for starting reservations.
"""
# Find starting/ending reservations
starting = self.slottable.get_reservations_starting_at(nowtime)
starting = [res for res in starting if res.state == ResourceReservation.STATE_SCHEDULED]
def schedule(self, nexttime):
""" The main scheduling function
The scheduling function looks at all pending requests and schedules them.
Note that most of the actual scheduling code is contained in the
__schedule_lease method and in the VMScheduler and PreparationScheduler classes.
Arguments:
nexttime -- The next time at which the scheduler can allocate resources.
"""
# Get pending leases
pending_leases = self.leases.get_leases_by_state(Lease.STATE_PENDING)
ar_leases = [
req for req in pending_leases
if req.get_type() == Lease.ADVANCE_RESERVATION
]
im_leases = [
req for req in pending_leases if req.get_type() == Lease.IMMEDIATE
]
be_leases = [
req for req in pending_leases
if req.get_type() == Lease.BEST_EFFORT
]
cognitive_leases = [
req for req in pending_leases if req.get_type() == Lease.COGNITIVE
]
#############################################################################################################################
#############################################################################################################################
# Second change (1): process cognitive leases directly #
#############################################################################################################################
#############################################################################################################################
for lease in cognitive_leases:
thread = Thread(target=self.Cognitive, args=(lease, nexttime))
thread.start()
thread.join()
#############################################################################################################################
#############################################################################################################################
# Queue best-effort leases
for lease in be_leases:
self.__enqueue(lease)
lease.set_state(Lease.STATE_QUEUED)
self.logger.info(
"Queued best-effort lease request #%i, %i nodes for %s." %
(lease.id, lease.numnodes, lease.duration.requested))
get_persistence().persist_lease(lease)
#Schedule immediate leases
for lease in im_leases:
self.logger.info("Scheduling immediate lease #%i (%i nodes)" %
(lease.id, lease.numnodes))
lease.print_contents()
try:
self.__schedule_lease(lease, nexttime=nexttime)
self.logger.info("Immediate lease #%i has been scheduled." %
lease.id)
lease.print_contents()
except NotSchedulableException, exc:
self.logger.info(
"Immediate lease request #%i cannot be scheduled: %s" %
(lease.id, exc.reason))
lease.set_state(Lease.STATE_REJECTED)
self.completed_leases.add(lease)
self.accounting.at_lease_done(lease)
self.leases.remove(lease)
get_persistence().persist_lease(lease)
def __schedule_lease(self, lease, nexttime):
""" Schedules a lease.
This method orchestrates the preparation and VM scheduler to
schedule a lease.
Arguments:
lease -- Lease to schedule.
nexttime -- The next time at which the scheduler can allocate resources.
"""
lease_state = lease.get_state()
migration = get_config().get("migration")
# Determine earliest start time in each node
if lease_state == Lease.STATE_PENDING or lease_state == Lease.STATE_QUEUED:
# This lease might require preparation. Ask the preparation
# scheduler for the earliest starting time.
earliest = self.preparation_scheduler.find_earliest_starting_times(lease, nexttime)
elif lease_state == Lease.STATE_SUSPENDED_PENDING or lease_state == Lease.STATE_SUSPENDED_QUEUED:
# This lease may have to be migrated.
# We have to ask both the preparation scheduler and the VM
# scheduler what would be the earliest possible starting time
# on each node, assuming we have to transfer files between
# nodes.
node_ids = self.slottable.nodes.keys()
earliest = {}
if migration == constants.MIGRATE_NO:
# If migration is disabled, the earliest starting time
# is simply nexttime.
for node in node_ids:
earliest[node] = EarliestStartingTime(nexttime, EarliestStartingTime.EARLIEST_NOPREPARATION)
else:
# Otherwise, we ask the preparation scheduler and the VM
# scheduler how long it would take them to migrate the
# lease state.
prep_migr_time = self.preparation_scheduler.estimate_migration_time(lease)
vm_migr_time = self.vm_scheduler.estimate_migration_time(lease)
for node in node_ids:
earliest[node] = EarliestStartingTime(nexttime + prep_migr_time + vm_migr_time, EarliestStartingTime.EARLIEST_MIGRATION)
else:
raise InconsistentLeaseStateError(lease, doing = "scheduling a best-effort lease")
# Now, we give the lease to the VM scheduler, along with the
# earliest possible starting times. If the VM scheduler can
# schedule VMs for this lease, it will return a resource reservation
# that we can add to the slot table, along with a list of
# leases that have to be preempted.
# If the VM scheduler can't schedule the VMs, it will throw an
# exception (we don't catch it here, and it is just thrown up
# to the calling method.
(vmrr, preemptions) = self.vm_scheduler.schedule(lease, nexttime, earliest)
# If scheduling the lease involves preempting other leases,
# go ahead and preempt them.
if len(preemptions) > 0:
self.logger.info("Must preempt leases %s to make room for lease #%i" % ([l.id for l in preemptions], lease.id))
for l in preemptions:
self.__preempt_lease(l, preemption_time=vmrr.start)
# Schedule lease preparation
is_ready = False
preparation_rrs = []
if lease_state in (Lease.STATE_SUSPENDED_PENDING, Lease.STATE_SUSPENDED_QUEUED) and migration != constants.MIGRATE_NO:
# The lease might require migration
migr_rrs = self.preparation_scheduler.schedule_migration(lease, vmrr, nexttime)
if len(migr_rrs) > 0:
end_migr = migr_rrs[-1].end
else:
end_migr = nexttime
migr_rrs += self.vm_scheduler.schedule_migration(lease, vmrr, end_migr)
migr_rrs.reverse()
for migr_rr in migr_rrs:
vmrr.pre_rrs.insert(0, migr_rr)
if len(migr_rrs) == 0:
is_ready = True
elif lease_state in (Lease.STATE_SUSPENDED_PENDING, Lease.STATE_SUSPENDED_QUEUED) and migration == constants.MIGRATE_NO:
# No migration means the lease is ready
is_ready = True
elif lease_state in (Lease.STATE_PENDING, Lease.STATE_QUEUED):
# The lease might require initial preparation
preparation_rrs, is_ready = self.preparation_scheduler.schedule(lease, vmrr, earliest)
# At this point, the lease is feasible.
# Commit changes by adding RRs to lease and to slot table
# Add preparation RRs (if any) to lease
for rr in preparation_rrs:
lease.append_preparationrr(rr)
# Add VMRR to lease
lease.append_vmrr(vmrr)
# Add resource reservations to slottable
# Preparation RRs (if any)
for rr in preparation_rrs:
self.slottable.add_reservation(rr)
# Pre-VM RRs (if any)
for rr in vmrr.pre_rrs:
self.slottable.add_reservation(rr)
# VM
self.slottable.add_reservation(vmrr)
# Post-VM RRs (if any)
for rr in vmrr.post_rrs:
self.slottable.add_reservation(rr)
# Change lease state
if lease_state == Lease.STATE_PENDING or lease_state == Lease.STATE_QUEUED:
lease.set_state(Lease.STATE_SCHEDULED)
if is_ready:
lease.set_state(Lease.STATE_READY)
elif lease_state == Lease.STATE_SUSPENDED_PENDING or lease_state == Lease.STATE_SUSPENDED_QUEUED:
lease.set_state(Lease.STATE_SUSPENDED_SCHEDULED)
get_persistence().persist_lease(lease)
lease.print_contents()
lease.start.requested + lease.duration.requested))
lease.print_contents()
try:
self.__schedule_lease(lease, nexttime)
self.logger.info("AR lease #%i has been scheduled." % lease.id)
lease.print_contents()
except NotSchedulableException, exc:
self.logger.info(
"AR lease request #%i cannot be scheduled: %s" %
(lease.id, exc.reason))
lease.set_state(Lease.STATE_REJECTED)
self.completed_leases.add(lease)
self.accounting.at_lease_done(lease)
self.leases.remove(lease)
get_persistence().persist_lease(lease)
# Process queue (i.e., traverse queue in search of leases that can be scheduled)
self.__process_queue(nexttime)
get_persistence().persist_queue(self.queue)
def process_starting_reservations(self, nowtime):
"""Processes starting reservations
This method checks the slottable to see if there are any reservations that are
starting at "nowtime". If so, the appropriate handler is called.
Arguments:
nowtime -- Time at which to check for starting reservations.
"""