def add_reservation(self, *, reservation: ABCReservationMixin, cycle: int):
"""
Adds a reservation associated with a given cycle.
@params reservation: the reservation
@params cycle: the cycle with which to associate the reservation
"""
if reservation is None or cycle < 0 or reservation.get_reservation_id(
) is None:
raise FrameworkException(Constants.INVALID_ARGUMENT)
if reservation.get_reservation_id() in self.reservation_id_to_cycle:
existing_cycle = self.reservation_id_to_cycle[
reservation.get_reservation_id()]
if existing_cycle == cycle:
return
else:
raise RuntimeError(
"Reservation: #{} is already in the list at a different cycle. Please remove it first, "
"before adding to a different cycle".format(
reservation.get_reservation_id()))
reservation_set = None
if cycle in self.cycle_to_rset:
reservation_set = self.cycle_to_rset[cycle]
else:
reservation_set = ReservationSet()
if reservation_set.contains(reservation=reservation) is False:
reservation_set.add(reservation=reservation)
self.count += 1
if cycle not in self.cycle_to_rset:
self.add_to_list(reservation_set=reservation_set, cycle=cycle)
self.cycle_to_rset[cycle] = reservation_set
self.reservation_id_to_cycle[reservation.get_reservation_id()] = cycle
def check_set(self, rset: ReservationSet, check: ReservationSet):
self.assertIsNotNone(check)
self.assertEqual(rset.size(), check.size())
for res in rset.values():
self.assertTrue(check.contains(reservation=res))
class ControllerCalendarPolicy(Policy, ABCControllerPolicy):
"""
The base class for all calendar-based service manager policy implementations.
"""
def __init__(self):
super().__init__()
# calendar
self.calendar = None
# Contains reservations for which we may have completed performing
# bookkeeping actions but may need to wait for some other event to take
# place before we raise the corresponding event.
self.pending_notify = ReservationSet()
# If the actor is initialized
self.initialized = False
# If true, the orchestrator will close reservations lazily: it will not
# issue a close and will wait until the site terminates the lease. The
# major drawback is that leave actions will not be able to connect to the
# resources, since the resources will not exist at this time.
self.lazy_close = False
def check_pending(self):
"""
Checks pending operations, and installs successfully completed
requests in the holdings calendar. Note that the policy module must add
bids to the pending set, or they may not install in the calendar.
@raises Exception in case of error
"""
rvset = self.calendar.get_pending()
if rvset is None:
return
for reservation in rvset.values():
if reservation.is_failed():
# This reservation has failed. Remove it from the list. This is
# a separate case, because we may fail but not satisfy the
# condition of the else statement.
self.logger.debug(
"Removing failed reservation from the pending list: {}".
format(reservation))
self.calendar.remove_pending(reservation=reservation)
self.pending_notify.remove(reservation=reservation)
elif reservation.is_no_pending(
) and not reservation.is_pending_recover():
# No pending operation and we are not about the reissue a recovery operation on this reservation.
self.logger.debug(
"Controller pending request completed {}".format(
reservation))
if reservation.is_closed():
# do nothing handled by close(IReservation)
self.logger.debug("No op")
elif reservation.is_active_ticketed():
# An active reservation extended its ticket.
# cancel the current close
self.calendar.remove_closing(reservation=reservation)
# schedule a new close
self.calendar.add_closing(reservation=reservation,
cycle=self.get_close(
reservation=reservation,
term=reservation.get_term()))
# Add from start to end instead of close. It is possible
# that holdings may not accurately reflect the actual
# number of resources towards the end of a lease. This is
# because we assume that we still have resources even after
# an advanceClose. When looking at this value, see if the
# reservation has closed.
self.calendar.add_holdings(
reservation=reservation,
start=reservation.get_term().get_new_start_time(),
end=reservation.get_term().get_end_time())
self.calendar.add_redeeming(
reservation=reservation,
cycle=self.get_redeem(reservation=reservation))
if reservation.is_renewable():
cycle = self.get_renew(reservation=reservation)
reservation.set_renew_time(time=cycle)
reservation.set_dirty()
self.calendar.add_renewing(reservation=reservation,
cycle=cycle)
self.pending_notify.remove(reservation=reservation)
elif reservation.is_ticketed():
# The reservation obtained a ticket for the first time
self.calendar.add_holdings(
reservation=reservation,
start=reservation.get_term().get_new_start_time(),
end=reservation.get_term().get_end_time())
self.calendar.add_redeeming(
reservation=reservation,
cycle=self.get_redeem(reservation=reservation))
self.calendar.add_closing(reservation=reservation,
cycle=self.get_close(
reservation=reservation,
term=reservation.get_term()))
if reservation.is_renewable():
cycle = self.get_renew(reservation=reservation)
reservation.set_renew_time(time=cycle)
reservation.set_dirty()
self.calendar.add_renewing(reservation=reservation,
cycle=cycle)
self.pending_notify.remove(reservation=reservation)
elif reservation.get_state() == ReservationStates.Active:
if self.pending_notify.contains(reservation=reservation):
# We are waiting for transfer in operations to complete
# so that we can raise the lease complete event.
if reservation.is_active_joined():
self.pending_notify.remove(reservation=reservation)
else:
# Just completed a lease call (redeem or extendLease).
# We need to remove this reservation from closing,
# because we added it using r.getTerm(), and add this
# reservation to closing using r.getLeasedTerm() [the
# site could have changed the term of the reservation].
# This assumes that r.getTerm has not changed in the
# mean time. This is true now, since the state machine
# does not allow more than one pending operation.
# Should we change this, we will need to update the
# code below.
self.calendar.remove_closing(reservation=reservation)
self.calendar.add_closing(
reservation=reservation,
cycle=self.get_close(
reservation=reservation,
term=reservation.get_lease_term()))
if reservation.get_renew_time() == 0:
reservation.set_renew_time(
time=(self.actor.get_current_cycle() + 1))
reservation.set_dirty()
self.calendar.add_renewing(
reservation=reservation,
cycle=reservation.get_renew_time())
if not reservation.is_active_joined():
# add to the pending notify list so that we can raise the event
# when transfer in operations complete.
self.pending_notify.add(reservation=reservation)
elif reservation.get_state() == ReservationStates.CloseWait or \
reservation.get_state() == ReservationStates.Failed:
self.pending_notify.remove(reservation=reservation)
else:
self.logger.warning(
"Invalid state on reservation. We may be still recovering: {}"
.format(reservation))
continue
if not self.pending_notify.contains(reservation=reservation):
self.logger.debug(
"Removing from pending: {}".format(reservation))
self.calendar.remove_pending(reservation=reservation)
def close(self, *, reservation: ABCReservationMixin):
# ignore any scheduled/in progress operations
self.calendar.remove_scheduled_or_in_progress(reservation=reservation)
def closed(self, *, reservation: ABCReservationMixin):
# remove the reservation from all calendar structures
self.calendar.remove_holdings(reservation=reservation)
self.calendar.remove_redeeming(reservation=reservation)
self.calendar.remove_renewing(reservation=reservation)
self.calendar.remove_closing(reservation=reservation)
self.pending_notify.remove(reservation=reservation)
def demand(self, *, reservation: ABCClientReservation):
if not reservation.is_nascent():
self.logger.error("demand reservation is not fresh")
else:
self.calendar.add_demand(reservation=reservation)
def extend(self, *, reservation: ABCReservationMixin,
resources: ResourceSet, term: Term):
# cancel any previously scheduled extends
self.calendar.remove_renewing(reservation=reservation)
# do not cancel the close: the extend may fail cancel any pending redeem: we will redeem after the extension
self.calendar.remove_redeeming(reservation=reservation)
# There should be no pending operations for this reservation at this time
# Add to the pending list so that we can track the progress of the reservation
self.calendar.add_pending(reservation=reservation)
def finish(self, *, cycle: int):
super().finish(cycle=cycle)
self.calendar.tick(cycle=cycle)
@abstractmethod
def get_close(self, *, reservation: ABCClientReservation,
term: Term) -> int:
"""
Returns the time that a reservation should be closed.
@params reservation reservation
@params term term
@returns the close time of the reservation (cycle)
@raises Exception in case of error
"""
def get_closing(self, *, cycle: int) -> ReservationSet:
closing = self.calendar.get_closing(cycle=cycle)
result = ReservationSet()
for reservation in closing.values():
if not reservation.is_failed():
self.calendar.add_pending(reservation=reservation)
result.add(reservation=reservation)
else:
self.logger.warning(
"Removing failed reservation from the closing list: {}".
format(reservation))
return result
@abstractmethod
def get_redeem(self, *, reservation: ABCClientReservation) -> int:
"""
Returns the time when the reservation should be redeemed.
@params reservation the reservation
@returns the redeem time of the reservation (cycle)
@raises Exception in case of error
"""
def get_redeeming(self, *, cycle: int) -> ReservationSet:
redeeming = self.calendar.get_redeeming(cycle=cycle)
for reservation in redeeming.values():
self.calendar.add_pending(reservation=reservation)
return redeeming
@abstractmethod
def get_renew(self, *, reservation: ABCClientReservation) -> int:
"""
Returns the time when the reservation should be renewed.
@params reservation the reservation
@returns the renew time of the reservation (cycle)
@raises Exception in case of error
"""
def initialize(self):
if not self.initialized:
super().initialize()
self.calendar = ControllerCalendar(clock=self.clock)
self.initialized = True
def is_expired(self, *, reservation: ABCReservationMixin):
"""
Checks if the reservation has expired.
@params reservation reservation to check
@returns true or false
"""
term = reservation.get_term()
end = self.clock.cycle(when=term.get_end_time())
return self.actor.get_current_cycle() > end
def remove(self, *, reservation: ABCReservationMixin):
# remove the reservation from the calendar
self.calendar.remove(reservation=reservation)
def revisit(self, *, reservation: ABCReservationMixin):
super().revisit(reservation=reservation)
if reservation.get_state() == ReservationStates.Nascent:
self.calendar.add_pending(reservation=reservation)
elif reservation.get_state() == ReservationStates.Ticketed:
if reservation.get_pending_state(
) == ReservationPendingStates.None_:
if reservation.is_pending_recover():
self.calendar.add_pending(reservation=reservation)
else:
self.calendar.add_redeeming(
reservation=reservation,
cycle=self.get_redeem(reservation=reservation))
self.calendar.add_holdings(
reservation=reservation,
start=reservation.get_term().get_new_start_time(),
end=reservation.get_term().get_end_time())
self.calendar.add_closing(reservation=reservation,
cycle=self.get_close(
reservation=reservation,
term=reservation.get_term()))
if reservation.is_renewable(
) and reservation.get_renew_time() != 0:
# Scheduling renewal is a bit tricky, since it may
# involve communication with the upstream broker.
# However, in some recovery cases, typical in one
# container deployment, the broker and the service
# manager will be recovering at the same time. In
# this case the query may fail and we will have to
# fail the reservation.
# Our approach here is as follows: we cache the
# renew time in the reservation class and persist
# it in the database. When we recover, we will
# check the renewTime field of the reservation if
# it is non-zero, we will use it, otherwise we will
# schedule the renew after we get the lease back
# from the authority.
self.calendar.add_renewing(
reservation=reservation,
cycle=reservation.get_renew_time())
elif reservation.get_pending_state(
) == ReservationPendingStates.Redeeming:
raise ControllerException(Constants.INVALID_RECOVERY_STATE)
elif reservation.get_state() == ReservationStates.Active:
if reservation.get_pending_state(
) == ReservationPendingStates.None_:
# pending list
if reservation.is_pending_recover():
self.calendar.add_pending(reservation=reservation)
# renewing
if reservation.is_renewable():
self.calendar.add_renewing(
reservation=reservation,
cycle=reservation.get_renew_time())
# holdings
self.calendar.add_holdings(
reservation=reservation,
start=reservation.get_term().get_new_start_time(),
end=reservation.get_term().get_end_time())
# closing
self.calendar.add_closing(
reservation=reservation,
cycle=self.get_close(reservation=reservation,
term=reservation.get_lease_term()))
elif reservation.get_pending_state(
) == ReservationPendingStates.ExtendingTicket:
raise ControllerException(Constants.INVALID_RECOVERY_STATE)
elif reservation.get_state() == ReservationStates.ActiveTicketed:
if reservation.get_pending_state(
) == ReservationPendingStates.None_:
if reservation.is_pending_recover():
self.calendar.add_pending(reservation=reservation)
else:
self.calendar.add_redeeming(
reservation=reservation,
cycle=self.get_redeem(reservation=reservation))
# holdings
self.calendar.add_holdings(
reservation=reservation,
start=reservation.get_term().get_new_start_time(),
end=reservation.get_term().get_end_time())
# closing
self.calendar.add_closing(
reservation=reservation,
cycle=self.get_close(reservation=reservation,
term=reservation.get_lease_term()))
# renewing
if reservation.is_renewable():
self.calendar.add_renewing(
reservation=reservation,
cycle=reservation.get_renew_time())
elif reservation.get_pending_state(
) == ReservationPendingStates.ExtendingLease:
raise ControllerException(Constants.INVALID_RECOVERY_STATE)
def ticket_satisfies(self, *, requested_resources: ResourceSet,
actual_resources: ResourceSet, requested_term: Term,
actual_term: Term):
return
def update_ticket_complete(self, *, reservation: ABCClientReservation):
return
def update_delegation_complete(self,
*,
delegation: ABCDelegation,
reclaim: bool = False):
return
def lease_satisfies(self, *, request_resources: ResourceSet,
actual_resources: ResourceSet, requested_term: Term,
actual_term: Term):
return
class Slice(ABCKernelSlice):
"""
Slice implementation. A slice has a globally unique identifier, name,
description, property list, an owning identity, an access control list, and a
set of reservations.
This class is used within the Service Manager, which may hold reservations on
many sites; on the Broker, which may have provided tickets to the slice for
reservations at many sites; and on the site Authority, where each slice may
hold multiple reservations for resources at that site.
"""
def __init__(self, *, slice_id: ID = None, name: str = "unspecified"):
# Globally unique identifier.
self.guid = slice_id
# Slice name. Not required to be globally or locally unique.
self.name = name
# Description string. Has only local meaning.
self.description = "no description"
# The slice type: inventory or client.
self.type = SliceTypes.ClientSlice
# The owner of the slice.
self.owner = None
# Resource type associated with this slice. Used when the slice is used to
# represent an inventory pool.
self.resource_type = None
# The reservations in this slice.
self.reservations = ReservationSet()
self.delegations = {}
# Neo4jGraph Id
self.graph_id = None
self.graph = None
self.state_machine = SliceStateMachine(slice_id=slice_id)
self.dirty = False
self.config_properties = None
self.lock = threading.Lock()
self.lease_end = None
self.lease_start = None
def __getstate__(self):
state = self.__dict__.copy()
del state['reservations']
del state['delegations']
del state['graph']
del state['lock']
return state
def __setstate__(self, state):
self.__dict__.update(state)
self.reservations = ReservationSet()
self.graph = None
self.delegations = {}
self.lock = threading.Lock()
def set_graph_id(self, graph_id: str):
self.graph_id = graph_id
def get_graph_id(self) -> str:
return self.graph_id
def set_graph(self, *, graph: ABCPropertyGraph):
self.graph = graph
self.set_graph_id(graph_id=self.graph.get_graph_id())
def get_graph(self) -> ABCPropertyGraph:
return self.graph
def clone_request(self) -> ABCSlice:
result = Slice()
result.name = self.name
result.guid = self.guid
return result
def get_lease_end(self) -> datetime:
return self.lease_end
def get_lease_start(self) -> datetime:
return self.lease_start
def get_description(self) -> str:
return self.description
def get_name(self) -> str:
return self.name
def get_owner(self) -> AuthToken:
return self.owner
def get_reservations(self) -> ReservationSet:
return self.reservations
def get_delegations(self) -> Dict[str, ABCDelegation]:
return self.delegations
def get_reservations_list(self) -> list:
return self.reservations.values()
def get_resource_type(self) -> ResourceType:
return self.resource_type
def get_slice_id(self) -> ID:
return self.guid
def is_broker_client(self) -> bool:
return self.type == SliceTypes.BrokerClientSlice
def is_client(self) -> bool:
return not self.is_inventory()
def is_inventory(self) -> bool:
return self.type == SliceTypes.InventorySlice
def is_empty(self) -> bool:
return self.reservations.is_empty()
def prepare(self):
self.reservations.clear()
self.delegations.clear()
self.state_machine.clear()
self.transition_slice(operation=SliceStateMachine.CREATE)
def register(self, *, reservation: ABCKernelReservation):
if self.reservations.contains(rid=reservation.get_reservation_id()):
raise SliceException(
"Reservation #{} already exists in slice".format(
reservation.get_reservation_id()))
self.reservations.add(reservation=reservation)
def register_delegation(self, *, delegation: ABCDelegation):
if delegation.get_delegation_id() in self.delegations:
raise SliceException(
"Delegation #{} already exists in slice".format(
delegation.get_delegation_id()))
self.delegations[delegation.get_delegation_id()] = delegation
def set_lease_end(self, *, lease_end: datetime):
self.lease_end = lease_end
def set_lease_start(self, *, lease_start: datetime):
self.lease_start = lease_start
def set_broker_client(self):
self.type = SliceTypes.BrokerClientSlice
def set_client(self):
self.type = SliceTypes.ClientSlice
def set_description(self, *, description: str):
self.description = description
def set_inventory(self, *, value: bool):
if value:
self.type = SliceTypes.InventorySlice
else:
self.type = SliceTypes.ClientSlice
def get_slice_type(self) -> SliceTypes:
return self.type
def set_name(self, *, name: str):
self.name = name
def set_owner(self, *, owner: AuthToken):
self.owner = owner
def set_resource_type(self, *, resource_type: ResourceType):
self.resource_type = resource_type
def soft_lookup(self, *, rid: ID) -> ABCKernelReservation:
return self.reservations.get(rid=rid)
def soft_lookup_delegation(self, *, did: str) -> ABCDelegation:
return self.delegations.get(did, None)
def __str__(self):
msg = "{}({})".format(self.name, str(self.guid))
if self.graph_id is not None:
msg += " Graph Id:{}".format(self.graph_id)
if self.owner is not None:
if self.owner.get_email() is not None:
msg += " Owner:{}".format(self.owner.get_email())
else:
msg += " Owner:{}".format(self.owner)
'''
if self.state_machine is not None:
msg += " State:{}".format(self.state_machine.get_state())
'''
return msg
def unregister(self, *, reservation: ABCKernelReservation):
self.reservations.remove(reservation=reservation)
def unregister_delegation(self, *, delegation: ABCDelegation):
if delegation.get_delegation_id() in self.delegations:
self.delegations.pop(delegation.get_delegation_id())
def get_state(self) -> SliceState:
return self.state_machine.get_state()
def set_dirty(self):
self.dirty = True
def clear_dirty(self):
self.dirty = False
def is_dirty(self) -> bool:
return self.dirty
def transition_slice(self, *,
operation: SliceOperation) -> Tuple[bool, SliceState]:
return self.state_machine.transition_slice(
operation=operation, reservations=self.reservations)
def is_stable_ok(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.StableOk:
return True
return False
def is_stable_error(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.StableError:
return True
return False
def is_stable(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.StableError or slice_state == SliceState.StableOk:
return True
return False
def is_dead_or_closing(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.Dead or slice_state == SliceState.Closing:
return True
return False
def is_dead(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.Dead:
return True
return False
def set_config_properties(self, *, value: dict):
self.config_properties = value
def get_config_properties(self) -> dict:
return self.config_properties
def update_slice_graph(self, sliver: BaseSliver, rid: str,
reservation_state: str) -> BaseSliver:
try:
self.lock.acquire()
# Update for Orchestrator for Active / Ticketed Reservations
if sliver is not None and self.graph_id is not None:
if sliver.reservation_info is None:
sliver.reservation_info = ReservationInfo()
sliver.reservation_info.reservation_id = rid
sliver.reservation_info.reservation_state = reservation_state
FimHelper.update_node(graph_id=self.graph_id, sliver=sliver)
return sliver
finally:
self.lock.release()