def map_growing(self, *, bids: ReservationSet):
"""
Maps reservations that are growing in this cycle (redeems or expanding
extends), and removes them from the bid set.
@param bids set of deferred operations for this cycle (non-null)
@throws Exception in case of error
"""
# self.logger.debug("Processing growing requests")
rids_to_remove = []
node_id_to_reservations = {}
for reservation in bids.values():
if reservation.is_terminal():
continue
adjust = reservation.get_deficit()
if adjust > 0:
if reservation.is_extending_lease():
self.logger.debug(
f"**Growing reservation by {adjust}:{reservation}")
else:
self.logger.debug(
f"**Redeeming reservation by {adjust}:{reservation}")
node_id_to_reservations = self.map(
reservation=reservation,
node_id_to_reservations=node_id_to_reservations)
rids_to_remove.append(reservation.get_reservation_id())
for rid in rids_to_remove:
bids.remove_by_rid(rid=rid)
def allocate_ticketing(self, *, requests: ReservationSet):
if requests is not None:
# Holds the Node Id to List of Reservation Ids allocated
# This is used to check on the reservations allocated during this cycle to compute available resources
# as the reservations are not updated in the database yet
node_id_to_reservations = {}
for reservation in requests.values():
if not reservation.is_ticketing():
continue
status, node_id_to_reservations, error_msg = self.ticket(
reservation=reservation,
node_id_to_reservations=node_id_to_reservations)
if status:
continue
if self.queue is None and not reservation.is_failed():
fail_message = "Insufficient resources"
if error_msg is not None:
fail_message = error_msg
reservation.fail(message=fail_message)
continue
if not reservation.is_failed():
fail_message = f"Insufficient resources for specified start time, Failing reservation: " \
f"{reservation.get_reservation_id()}"
if error_msg is not None:
fail_message = error_msg
reservation.fail(message=fail_message)
def map_shrinking(self, *, bids: ReservationSet):
"""
Maps reservations that are shrinking or staying the same (extending with
no flex) in this cycle, and removes them from the bid set.
@param bids set of deferred operations for this cycle (non-null)
@raises Exception in case of error
"""
# self.logger.debug("Processing shrinking requests")
rids_to_remove = []
node_id_to_reservations = {}
for reservation in bids.values():
adjust = reservation.get_deficit()
if adjust > 0:
continue
if not reservation.is_terminal(
) and reservation.is_extending_lease():
if adjust < 0:
self.logger.debug(
f"**Shrinking reservation by {adjust}:{reservation}")
else:
self.logger.debug(
f"**Extending reservation (no flex): {reservation}")
node_id_to_reservations = self.map(
reservation=reservation,
node_id_to_reservations=node_id_to_reservations)
rids_to_remove.append(reservation.get_reservation_id())
for rid in rids_to_remove:
bids.remove_by_rid(rid=rid)
def extend_lease(self,
*,
reservation: ABCControllerReservation = None,
rset: ReservationSet = None):
if reservation is not None and rset is not None:
raise ControllerException(
"Invalid Arguments: reservation and rset can not be both not None"
)
if reservation is None and rset is None:
raise ControllerException(
"Invalid Arguments: reservation and rset can not be both None")
if reservation is not None:
self.extend_lease_reservation(reservation=reservation)
if rset is not None:
for r in rset.values():
try:
if isinstance(r, ABCControllerReservation):
self.extend_lease_reservation(reservation=r)
else:
self.logger.warning(
"Reservation #{} cannot extendLease".format(
r.get_reservation_id()))
except Exception as e:
self.logger.error(
"Could not extend_lease for #{} e={}".format(
r.get_reservation_id(), e))
def close_reservations(self, *, reservations: ReservationSet):
for reservation in reservations.values():
try:
self.logger.debug("Closing reservation: {}".format(
reservation.get_reservation_id()))
self.close(reservation=reservation)
except Exception as e:
self.logger.error(traceback.format_exc())
self.logger.error("Could not close for #{} {}".format(
reservation.get_reservation_id(), e))
def extend_lease_reservations(self, *, rset: ReservationSet):
"""
Extend all reservations:
@param rset: reservation set
"""
for reservation in rset.values():
try:
self.extend_lease(reservation=reservation)
except Exception as e:
self.logger.error("Could not redeem for # {} {}".format(reservation.get_reservation_id(), e))
def tickets_client(self, *, rset: ReservationSet):
for reservation in rset.values():
try:
if isinstance(reservation, ABCBrokerReservation):
self.ticket_broker(reservation=reservation)
elif isinstance(reservation, ABCClientReservation):
self.ticket_client(reservation=reservation)
else:
self.logger.warning("Reservation #{} cannot be ticketed".format(reservation.get_reservation_id()))
except Exception as e:
self.logger.error("Could not ticket for #{} e: {}".format(reservation.get_reservation_id(), e))
def redeem_reservations(self, *, rset: ReservationSet):
"""
Redeem all reservations:
@param rset: reservation set
"""
for reservation in rset.values():
try:
if isinstance(reservation, ABCAuthorityReservation):
self.redeem(reservation=reservation)
else:
self.logger.warning("Reservation # {} cannot be redeemed".format(reservation.get_reservation_id()))
except Exception as e:
self.logger.error("Could not redeem for # {} {}".format(reservation.get_reservation_id(), e))
def redeem_reservations(self, *, rset: ReservationSet):
for reservation in rset.values():
try:
if isinstance(reservation, ABCControllerReservation):
self.redeem(reservation=reservation)
else:
self.logger.warning(
"Reservation #{} cannot be redeemed".format(
reservation.get_reservation_id()))
except Exception as e:
self.logger.error(traceback.format_exc())
self.logger.error("Could not redeem for #{} {}".format(
reservation.get_reservation_id(), e))
def all_failed(*, reservations: ReservationSet) -> bool:
"""
We don't introduce a special state to flag when a slice is ALL FAILED, however this helper function helps decide
when to GC a slice
@return true or false
"""
bins = StateBins()
for r in reservations.values():
bins.add(s=r.get_state())
if not bins.has_state_other_than(ReservationStates.Failed):
return True
return False
def process_renewing(self, *, renewing: ReservationSet) -> ReservationSet:
"""
Performs checks on renewing reservations. Updates the terms to
suggest new terms, stores the extend on the pending list. Returns a
fresh ReservationSet of expiring reservations to try to renew in this
bidding cycle.
@param renewing collection of the renewing reservations
@return non-null set of renewals
"""
result = ReservationSet()
if renewing is None:
return None
#self.logger.debug("Expiring = {}".format(renewing.size()))
for reservation in renewing.values():
self.logger.debug("Expiring res: {}".format(reservation))
if reservation.is_renewable():
self.logger.debug("This is a renewable expiring reservation")
term = reservation.get_term()
term = term.extend()
reservation.set_approved(term=term,
approved_resources=reservation.
get_resources().abstract_clone())
result.add(reservation=reservation)
self.calendar.add_pending(reservation=reservation)
else:
self.logger.debug(
"This is not a renewable expiring reservation")
return result
def allocate_extending_reservation_set(self, *, requests: ReservationSet):
if requests is not None:
for reservation in requests.values():
if reservation.is_extending_ticket(
) and not reservation.is_closed():
start = reservation.get_requested_term(
).get_new_start_time()
end = self.align_end(
when=reservation.get_requested_term().get_end_time())
resource_type = reservation.get_resources().get_type()
inv = self.inventory.get(resource_type=resource_type)
if inv is not None:
ext_term = Term(
start=reservation.get_term().get_start_time(),
end=end,
new_start=start)
self.extend_private(reservation=reservation,
inv=inv,
term=ext_term)
else:
reservation.fail(message=Constants.NO_POOL)
def check_set(self, rset: ReservationSet, check: ReservationSet):
self.assertIsNotNone(check)
self.assertEqual(rset.size(), check.size())
for res in rset.values():
self.assertTrue(check.contains(reservation=res))
class ControllerTicketReviewPolicy(ControllerSimplePolicy):
"""
This implementation of a Controller policy is almost identical to the parent ControllerSimplePolicy.
The only real difference is that it addresses that Tickets should not be redeemed if any reservations are
currently Failed or Nascent. This effectively acts as a "gate" between the Controller and AM.
All reservations must be Ticketed, before any reservations are allowed to be redeemed.
"""
def __init__(self):
super().__init__()
self.pending_redeem = ReservationSet()
def __getstate__(self):
state = self.__dict__.copy()
del state['logger']
del state['actor']
del state['clock']
del state['initialized']
del state['pending_notify']
del state['lazy_close']
del state['pending_redeem']
return state
def __setstate__(self, state):
self.__dict__.update(state)
self.logger = None
self.actor = None
self.clock = None
self.initialized = False
self.pending_notify = ReservationSet()
self.lazy_close = False
self.pending_redeem = ReservationSet()
def check_pending(self):
"""
Check to make sure all reservations are Ticketed (not Failed or Nascent)
before calling the parent method.
@throws Exception in case of error
"""
# add all of our pendingRedeem, so they can be checked
for reservation in self.pending_redeem.values():
self.calendar.add_pending(reservation=reservation)
# get set of reservations that need to be redeemed
my_pending = self.calendar.get_pending()
# keep track of status of the slice containing each reservation
slice_status_map = {}
# nothing to do!
if my_pending is None:
return
# check the status of the Slice of each reservation
for reservation in my_pending.values():
slice_obj = reservation.get_slice()
slice_id = slice_obj.get_slice_id()
# only want to do this for 'new' tickets
if reservation.is_failed() or reservation.is_ticketed():
# check if we've examined this slice already
if slice_id not in slice_status_map:
# set the default status
slice_status_map[
slice_id] = TicketReviewSliceState.Redeemable
# examine every reservation contained within the slice,
# looking for either a Failed or Nascent reservation
# we have to look at everything in a slice once, to determine all/any Sites with failures
for slice_reservation in slice_obj.get_reservations(
).values():
# If any Reservations that are being redeemed, that means the
# slice has already cleared TicketReview.
if slice_reservation.is_redeeming():
if slice_status_map[
slice_id] == TicketReviewSliceState.Nascent:
# There shouldn't be any Nascent reservations, if a reservation is being Redeemed.
self.logger.error(
"Nascent reservation found while Reservation {} in slice {} is redeeming"
.format(
slice_reservation.get_reservation_id(),
slice_obj.get_name()))
# We may have previously found a Failed Reservation,
# but if a ticketed reservation is being redeemed,
# the failure _should_ be from the AM, not Controller
# so it should be ignored by TicketReview
slice_status_map[
slice_id] = TicketReviewSliceState.Redeemable
# we don't need to look at any other reservations in this slice
break
# if any tickets are Nascent,
# as soon as we remove the Failed reservation,
# those Nascent tickets might get redeemed.
# we must wait to Close any failed reservations
# until all Nascent tickets are either Ticketed or Failed
if slice_reservation.is_nascent():
self.logger.debug(
"Found Nascent Reservation {} in slice {} when check_pending for {}"
.format(slice_reservation.get_reservation_id(),
slice_obj.get_name(),
reservation.get_reservation_id()))
slice_status_map[
slice_id] = TicketReviewSliceState.Nascent
# once we have found a Nascent reservation, that is what we treat the entire slice
break
# track Failed reservations, but need to keep looking for Nascent or Redeemable.
if slice_reservation.is_failed():
self.logger.debug(
"Found failed reservation {} in slice {} when check_pending for {}"
.format(slice_reservation.get_reservation_id(),
slice_obj.get_name(),
reservation.get_reservation_id()))
slice_status_map[
slice_id] = TicketReviewSliceState.Failing
# take action on the current reservation
if slice_status_map[
slice_id] == TicketReviewSliceState.Failing:
if reservation.get_resources(
) is not None and reservation.get_resources().get_type(
) is not None:
msg = f"TicketReviewPolicy: Closing reservation {reservation.get_reservation_id()} due to " \
f"failure in slice {slice_obj.get_name()}"
self.logger.info(msg)
if not reservation.is_failed():
update_data = UpdateData()
update_data.failed = True
update_data.message = Constants.CLOSURE_BY_TICKET_REVIEW_POLICY
reservation.mark_close_by_ticket_review(
update_data=update_data)
self.actor.close(reservation=reservation)
self.calendar.remove_pending(reservation=reservation)
self.pending_notify.remove(reservation=reservation)
elif slice_status_map[
slice_id] == TicketReviewSliceState.Nascent:
self.logger.info(
"Moving reservation {} to pending redeem list due to nascent reservation in slice {}"
.format(reservation.get_reservation_id(),
slice_obj.get_name()))
self.pending_redeem.add(reservation=reservation)
self.calendar.remove_pending(reservation=reservation)
else:
# we don't need to look at any other reservations in this slice
self.logger.debug(
"Removing from pendingRedeem: {}".format(reservation))
self.pending_redeem.remove(reservation=reservation)
else:
# Remove active or close reservations
self.logger.debug(
"Removing from pendingRedeem: {}".format(reservation))
self.pending_redeem.remove(reservation=reservation)
super().check_pending()
def transition_slice(
self, *, operation: SliceOperation,
reservations: ReservationSet) -> Tuple[bool, SliceState]:
"""
Attempt to transition a slice to a new state
@param operation slice operation
@param reservations reservations
@return Slice State
@throws Exception in case of error
"""
state_changed = False
prev_state = self.state
if self.state not in operation.valid_from_states:
raise SliceException(
f"Operation: {operation} cannot transition from state {self.state}"
)
if operation.command == SliceCommand.Create:
self.state = SliceState.Configuring
elif operation.command == SliceCommand.Modify:
self.state = SliceState.Configuring
elif operation.command == SliceCommand.Delete:
if self.state != SliceState.Dead:
self.state = SliceState.Closing
elif operation.command == SliceCommand.Reevaluate:
if reservations is None or reservations.size() == 0:
return state_changed, self.state
bins = StateBins()
for r in reservations.values():
bins.add(s=r.get_state())
if self.state == SliceState.Nascent or self.state == SliceState.Configuring:
if not bins.has_state_other_than(ReservationStates.Active,
ReservationStates.Closed):
self.state = SliceState.StableOK
if (not bins.has_state_other_than(ReservationStates.Active, ReservationStates.Failed,
ReservationStates.Closed)) and \
bins.has_state(s=ReservationStates.Failed):
self.state = SliceState.StableError
if not bins.has_state_other_than(ReservationStates.Closed,
ReservationStates.CloseWait,
ReservationStates.Failed):
self.state = SliceState.Closing
elif self.state == SliceState.StableError or self.state == SliceState.StableOK:
if not bins.has_state_other_than(ReservationStates.Closed,
ReservationStates.CloseWait,
ReservationStates.Failed):
self.state = SliceState.Dead
if not bins.has_state_other_than(
ReservationStates.Closed, ReservationStates.CloseWait,
ReservationPendingStates.Closing,
ReservationStates.Failed):
self.state = SliceState.Closing
elif self.state == SliceState.Closing and not bins.has_state_other_than(
ReservationStates.CloseWait, ReservationStates.Closed,
ReservationStates.Failed):
self.state = SliceState.Dead
if prev_state != self.state:
state_changed = True
return state_changed, self.state
class Slice(ABCKernelSlice):
"""
Slice implementation. A slice has a globally unique identifier, name,
description, property list, an owning identity, an access control list, and a
set of reservations.
This class is used within the Service Manager, which may hold reservations on
many sites; on the Broker, which may have provided tickets to the slice for
reservations at many sites; and on the site Authority, where each slice may
hold multiple reservations for resources at that site.
"""
def __init__(self, *, slice_id: ID = None, name: str = "unspecified"):
# Globally unique identifier.
self.guid = slice_id
# Slice name. Not required to be globally or locally unique.
self.name = name
# Description string. Has only local meaning.
self.description = "no description"
# The slice type: inventory or client.
self.type = SliceTypes.ClientSlice
# The owner of the slice.
self.owner = None
# Resource type associated with this slice. Used when the slice is used to
# represent an inventory pool.
self.resource_type = None
# The reservations in this slice.
self.reservations = ReservationSet()
self.delegations = {}
# Neo4jGraph Id
self.graph_id = None
self.graph = None
self.state_machine = SliceStateMachine(slice_id=slice_id)
self.dirty = False
self.config_properties = None
self.lock = threading.Lock()
self.lease_end = None
self.lease_start = None
def __getstate__(self):
state = self.__dict__.copy()
del state['reservations']
del state['delegations']
del state['graph']
del state['lock']
return state
def __setstate__(self, state):
self.__dict__.update(state)
self.reservations = ReservationSet()
self.graph = None
self.delegations = {}
self.lock = threading.Lock()
def set_graph_id(self, graph_id: str):
self.graph_id = graph_id
def get_graph_id(self) -> str:
return self.graph_id
def set_graph(self, *, graph: ABCPropertyGraph):
self.graph = graph
self.set_graph_id(graph_id=self.graph.get_graph_id())
def get_graph(self) -> ABCPropertyGraph:
return self.graph
def clone_request(self) -> ABCSlice:
result = Slice()
result.name = self.name
result.guid = self.guid
return result
def get_lease_end(self) -> datetime:
return self.lease_end
def get_lease_start(self) -> datetime:
return self.lease_start
def get_description(self) -> str:
return self.description
def get_name(self) -> str:
return self.name
def get_owner(self) -> AuthToken:
return self.owner
def get_reservations(self) -> ReservationSet:
return self.reservations
def get_delegations(self) -> Dict[str, ABCDelegation]:
return self.delegations
def get_reservations_list(self) -> list:
return self.reservations.values()
def get_resource_type(self) -> ResourceType:
return self.resource_type
def get_slice_id(self) -> ID:
return self.guid
def is_broker_client(self) -> bool:
return self.type == SliceTypes.BrokerClientSlice
def is_client(self) -> bool:
return not self.is_inventory()
def is_inventory(self) -> bool:
return self.type == SliceTypes.InventorySlice
def is_empty(self) -> bool:
return self.reservations.is_empty()
def prepare(self):
self.reservations.clear()
self.delegations.clear()
self.state_machine.clear()
self.transition_slice(operation=SliceStateMachine.CREATE)
def register(self, *, reservation: ABCKernelReservation):
if self.reservations.contains(rid=reservation.get_reservation_id()):
raise SliceException(
"Reservation #{} already exists in slice".format(
reservation.get_reservation_id()))
self.reservations.add(reservation=reservation)
def register_delegation(self, *, delegation: ABCDelegation):
if delegation.get_delegation_id() in self.delegations:
raise SliceException(
"Delegation #{} already exists in slice".format(
delegation.get_delegation_id()))
self.delegations[delegation.get_delegation_id()] = delegation
def set_lease_end(self, *, lease_end: datetime):
self.lease_end = lease_end
def set_lease_start(self, *, lease_start: datetime):
self.lease_start = lease_start
def set_broker_client(self):
self.type = SliceTypes.BrokerClientSlice
def set_client(self):
self.type = SliceTypes.ClientSlice
def set_description(self, *, description: str):
self.description = description
def set_inventory(self, *, value: bool):
if value:
self.type = SliceTypes.InventorySlice
else:
self.type = SliceTypes.ClientSlice
def get_slice_type(self) -> SliceTypes:
return self.type
def set_name(self, *, name: str):
self.name = name
def set_owner(self, *, owner: AuthToken):
self.owner = owner
def set_resource_type(self, *, resource_type: ResourceType):
self.resource_type = resource_type
def soft_lookup(self, *, rid: ID) -> ABCKernelReservation:
return self.reservations.get(rid=rid)
def soft_lookup_delegation(self, *, did: str) -> ABCDelegation:
return self.delegations.get(did, None)
def __str__(self):
msg = "{}({})".format(self.name, str(self.guid))
if self.graph_id is not None:
msg += " Graph Id:{}".format(self.graph_id)
if self.owner is not None:
if self.owner.get_email() is not None:
msg += " Owner:{}".format(self.owner.get_email())
else:
msg += " Owner:{}".format(self.owner)
'''
if self.state_machine is not None:
msg += " State:{}".format(self.state_machine.get_state())
'''
return msg
def unregister(self, *, reservation: ABCKernelReservation):
self.reservations.remove(reservation=reservation)
def unregister_delegation(self, *, delegation: ABCDelegation):
if delegation.get_delegation_id() in self.delegations:
self.delegations.pop(delegation.get_delegation_id())
def get_state(self) -> SliceState:
return self.state_machine.get_state()
def set_dirty(self):
self.dirty = True
def clear_dirty(self):
self.dirty = False
def is_dirty(self) -> bool:
return self.dirty
def transition_slice(self, *,
operation: SliceOperation) -> Tuple[bool, SliceState]:
return self.state_machine.transition_slice(
operation=operation, reservations=self.reservations)
def is_stable_ok(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.StableOk:
return True
return False
def is_stable_error(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.StableError:
return True
return False
def is_stable(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.StableError or slice_state == SliceState.StableOk:
return True
return False
def is_dead_or_closing(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.Dead or slice_state == SliceState.Closing:
return True
return False
def is_dead(self) -> bool:
state_changed, slice_state = self.transition_slice(
operation=SliceStateMachine.REEVALUATE)
if slice_state == SliceState.Dead:
return True
return False
def set_config_properties(self, *, value: dict):
self.config_properties = value
def get_config_properties(self) -> dict:
return self.config_properties
def update_slice_graph(self, sliver: BaseSliver, rid: str,
reservation_state: str) -> BaseSliver:
try:
self.lock.acquire()
# Update for Orchestrator for Active / Ticketed Reservations
if sliver is not None and self.graph_id is not None:
if sliver.reservation_info is None:
sliver.reservation_info = ReservationInfo()
sliver.reservation_info.reservation_id = rid
sliver.reservation_info.reservation_state = reservation_state
FimHelper.update_node(graph_id=self.graph_id, sliver=sliver)
return sliver
finally:
self.lock.release()
class Controller(ActorMixin, ABCController):
"""
Implements Controller
"""
saved_extended_renewable = ReservationSet()
def __init__(self,
*,
identity: AuthToken = None,
clock: ActorClock = None):
super().__init__(auth=identity, clock=clock)
# Recovered reservations that need to obtain tickets.
self.ticketing = ReservationSet()
# Recovered reservations that need to extend tickets.
self.extending_ticket = ReservationSet()
# Recovered reservations that need to be redeemed.
self.redeeming = ReservationSet()
# Recovered reservations that need to extend leases.
self.extending_lease = ReservationSet()
# Recovered reservations that need to modify leases
self.modifying_lease = ReservationSet()
# Peer registry.
self.registry = PeerRegistry()
# initialization status
self.initialized = False
self.type = ActorType.Orchestrator
def __getstate__(self):
state = self.__dict__.copy()
del state['recovered']
del state['wrapper']
del state['logger']
del state['clock']
del state['current_cycle']
del state['first_tick']
del state['stopped']
del state['initialized']
del state['thread_lock']
del state['thread']
del state['timer_queue']
del state['event_queue']
del state['reservation_tracker']
del state['subscription_id']
del state['actor_main_lock']
del state['closing']
del state['message_service']
del state['ticketing']
del state['extending_ticket']
del state['redeeming']
del state['extending_lease']
del state['modifying_lease']
del state['registry']
return state
def __setstate__(self, state):
self.__dict__.update(state)
self.recovered = False
self.wrapper = None
self.logger = None
self.clock = None
self.current_cycle = -1
self.first_tick = True
self.stopped = False
self.initialized = False
self.thread = None
self.thread_lock = threading.Lock()
self.timer_queue = queue.Queue()
self.event_queue = queue.Queue()
self.reservation_tracker = None
self.subscription_id = None
self.actor_main_lock = threading.Condition()
self.closing = ReservationSet()
self.message_service = None
self.ticketing = ReservationSet()
self.extending_ticket = ReservationSet()
self.redeeming = ReservationSet()
self.extending_lease = ReservationSet()
self.modifying_lease = ReservationSet()
self.registry = PeerRegistry()
def actor_added(self):
super().actor_added()
self.registry.actor_added()
def add_broker(self, *, broker: ABCBrokerProxy):
self.registry.add_broker(broker=broker)
def bid(self):
"""
Bids for resources as dictated by the plugin bidding policy for the
current cycle.
@throws Exception in case of error
"""
# Invoke policy module to select candidates for ticket and extend. Note
# that candidates structure is discarded when we're done.
candidates = self.policy.formulate_bids(cycle=self.current_cycle)
if candidates is not None:
# Issue new ticket requests.
ticketing = candidates.get_ticketing()
if ticketing is not None:
for ticket in ticketing.values():
try:
self.wrapper.ticket(reservation=ticket,
destination=self)
except Exception as e:
self.logger.error(
"unexpected ticket failure for #{} {}".format(
ticket.get_reservation_id(), e))
ticket.fail(
message="unexpected ticket failure {}".format(e))
extending = candidates.get_extending()
if extending is not None:
for extend in extending.values():
try:
self.wrapper.extend_ticket(reservation=extend)
except Exception as e:
self.logger.error(
"unexpected extend failure for #{} {}".format(
extend.get_reservation_id(), e))
extend.fail(
message="unexpected extend failure {}".format(e))
def claim_delegation_client(self,
*,
delegation_id: str = None,
slice_object: ABCSlice = None,
broker: ABCBrokerProxy = None,
id_token: str = None) -> ABCDelegation:
raise ControllerException("Not implemented")
def reclaim_delegation_client(self,
*,
delegation_id: str = None,
slice_object: ABCSlice = None,
broker: ABCBrokerProxy = None,
id_token: str = None) -> ABCDelegation:
raise ControllerException("Not implemented")
def close_expiring(self):
"""
Issues close requests on all reservations scheduled for closing on the
current cycle
"""
rset = self.policy.get_closing(cycle=self.current_cycle)
if rset is not None and rset.size() > 0:
self.logger.info(
"SlottedSM close expiring for cycle {} expiring {}".format(
self.current_cycle, rset))
self.close_reservations(reservations=rset)
def demand(self, *, rid: ID):
if rid is None:
raise ControllerException("Invalid argument")
reservation = self.get_reservation(rid=rid)
if reservation is None:
raise ControllerException("Unknown reservation {}".format(rid))
self.policy.demand(reservation=reservation)
reservation.set_policy(policy=self.policy)
def extend_lease_reservation(self, *,
reservation: ABCControllerReservation):
"""
Extend Lease for a reservation
@param reservation reservation
"""
if not self.recovered:
self.extending_lease.add(reservation=reservation)
else:
self.wrapper.extend_lease(reservation=reservation)
def extend_lease(self,
*,
reservation: ABCControllerReservation = None,
rset: ReservationSet = None):
if reservation is not None and rset is not None:
raise ControllerException(
"Invalid Arguments: reservation and rset can not be both not None"
)
if reservation is None and rset is None:
raise ControllerException(
"Invalid Arguments: reservation and rset can not be both None")
if reservation is not None:
self.extend_lease_reservation(reservation=reservation)
if rset is not None:
for r in rset.values():
try:
if isinstance(r, ABCControllerReservation):
self.extend_lease_reservation(reservation=r)
else:
self.logger.warning(
"Reservation #{} cannot extendLease".format(
r.get_reservation_id()))
except Exception as e:
self.logger.error(
"Could not extend_lease for #{} e={}".format(
r.get_reservation_id(), e))
def extend_ticket_client(self, *, reservation: ABCClientReservation):
if not self.recovered:
self.extending_ticket.add(reservation=reservation)
else:
self.wrapper.extend_ticket(reservation=reservation)
def extend_tickets_client(self, *, rset: ReservationSet):
for reservation in rset.values():
try:
if isinstance(reservation, ABCClientReservation):
self.extend_ticket_client(reservation=reservation)
else:
self.logger.warning(
"Reservation # {} cannot be ticketed".format(
reservation.get_reservation_id()))
except Exception as e:
self.logger.error("Could not ticket for #{} e: {}".format(
reservation.get_reservation_id(), e))
def get_broker(self, *, guid: ID) -> ABCBrokerProxy:
return self.registry.get_broker(guid=guid)
def get_brokers(self) -> list:
return self.registry.get_brokers()
def get_default_broker(self) -> ABCBrokerProxy:
return self.registry.get_default_broker()
def initialize(self):
if not self.initialized:
super().initialize()
self.registry.set_slices_plugin(plugin=self.plugin)
self.registry.initialize()
self.initialized = True
def process_redeeming(self):
"""
Issue redeem requests on all reservations scheduled for redeeming on the current cycle
"""
rset = self.policy.get_redeeming(cycle=self.current_cycle)
if rset is not None and rset.size() > 0:
self.logger.info(
"SlottedController redeem for cycle {} redeeming {}".format(
self.current_cycle, rset))
self.redeem_reservations(rset=rset)
def redeem(self, *, reservation: ABCControllerReservation):
if not self.recovered:
self.redeeming.add(reservation=reservation)
else:
self.wrapper.redeem(reservation=reservation)
def redeem_reservations(self, *, rset: ReservationSet):
for reservation in rset.values():
try:
if isinstance(reservation, ABCControllerReservation):
self.redeem(reservation=reservation)
else:
self.logger.warning(
"Reservation #{} cannot be redeemed".format(
reservation.get_reservation_id()))
except Exception as e:
self.logger.error(traceback.format_exc())
self.logger.error("Could not redeem for #{} {}".format(
reservation.get_reservation_id(), e))
def ticket_client(self, *, reservation: ABCClientReservation):
if not self.recovered:
self.ticketing.add(reservation=reservation)
else:
self.wrapper.ticket(reservation=reservation, destination=self)
def tickets_client(self, *, rset: ReservationSet):
for reservation in rset.values():
try:
if isinstance(reservation, ABCClientReservation):
self.ticket_client(reservation=reservation)
else:
self.logger.warning(
"Reservation #{} cannot be ticketed".format(
reservation.get_reservation_id()))
except Exception as e:
self.logger.error("Could not ticket for #{} e: {}".format(
reservation.get_reservation_id(), e))
def tick_handler(self):
self.close_expiring()
self.process_redeeming()
self.bid()
def update_lease(self, *, reservation: ABCReservationMixin, update_data,
caller: AuthToken):
if not self.is_recovered() or self.is_stopped():
raise ControllerException("This actor cannot receive calls")
self.wrapper.update_lease(reservation=reservation,
update_data=update_data,
caller=caller)
def update_ticket(self, *, reservation: ABCReservationMixin, update_data,
caller: AuthToken):
if not self.is_recovered() or self.is_stopped():
raise ControllerException("This actor cannot receive calls")
self.wrapper.update_ticket(reservation=reservation,
update_data=update_data,
caller=caller)
def update_delegation(self, *, delegation: ABCDelegation, update_data,
caller: AuthToken):
raise ControllerException("Not supported in controller")
def modify(self, *, reservation_id: ID, modify_properties: dict):
if reservation_id is None or modify_properties is None:
self.logger.error(
"modifyProperties argument is null or non-existing reservation"
)
rc = None
try:
rc = self.get_reservation(rid=reservation_id)
except Exception as e:
self.logger.error("Could not find reservation #{} e: {}".format(
reservation_id, e))
if rc is None:
raise ControllerException(
"Unknown reservation: {}".format(reservation_id))
if rc.get_resources() is not None:
# TODO
print("TODO")
else:
self.logger.warning(
"There are no approved resources for {}, no modify properties will be added"
.format(reservation_id))
if not self.recovered:
self.modifying_lease.add(reservation=rc)
else:
self.wrapper.modify_lease(reservation=rc)
def save_extending_renewable(self):
"""
For recovery, mark extending reservations renewable or the opposite
and save this, then restore afterwards
"""
for reservation in self.extending_ticket.values():
try:
if isinstance(reservation, ABCClientReservation):
if not reservation.get_renewable():
reservation.set_renewable(renewable=True)
self.saved_extended_renewable.add(
reservation=reservation)
else:
self.logger.warning(
"Reservation #{} cannot be remarked".format(
reservation.get_reservation_id()))
except Exception as e:
self.logger.error(
"Could not mark ticket renewable for #{} e: {}".format(
reservation.get_reservation_id(), e))
def restore_extending_renewable(self):
"""
Restore the value of renewable field after recovery if we changed it
"""
for reservation in self.saved_extended_renewable.values():
try:
reservation.set_renewable(renewable=False)
except Exception as e:
self.logger.error(
"Could not remark ticket non renewable for #{} e: {}".
format(reservation.get_reservation_id(), e))
def issue_delayed(self):
super().issue_delayed()
self.tickets_client(rset=self.ticketing)
self.ticketing.clear()
self.save_extending_renewable()
self.extend_tickets_client(rset=self.extending_ticket)
self.extending_ticket.clear()
self.redeem_reservations(rset=self.redeeming)
self.redeeming.clear()
self.extend_lease(rset=self.extending_lease)
self.extending_lease.clear()
@staticmethod
def get_management_object_class() -> str:
return ControllerManagementObject.__name__
@staticmethod
def get_management_object_module() -> str:
return ControllerManagementObject.__module__
@staticmethod
def get_kafka_service_class() -> str:
return ControllerService.__name__
@staticmethod
def get_kafka_service_module() -> str:
return ControllerService.__module__
@staticmethod
def get_mgmt_kafka_service_class() -> str:
return KafkaControllerService.__name__
@staticmethod
def get_mgmt_kafka_service_module() -> str:
return KafkaControllerService.__module__