Esempio n. 1
0
    def test_demand(self):
        cal = self._get_calendar()
        rset = ReservationSet()

        for i in range(5):
            r = self._make_reservation(id=str(i))
            # add to the list
            rset.add(reservation=r)
            cal.add_demand(reservation=r)
            # get the list and check it
            temp = cal.get_demand()
            self.check_set(rset=rset, check=temp)
            # remove from the returned set
            temp.remove(reservation=r)
            # make sure this did not affect the parent data structure
            temp = cal.get_demand()
            self.check_set(rset=rset, check=temp)

        for i in range(5):
            r = self._make_reservation(id=str(i))
            # add to the list
            rset.remove(reservation=r)
            cal.remove_demand(reservation=r)

            # get the list and check it
            temp = cal.get_demand()

            self.check_set(rset=rset, check=temp)
Esempio n. 2
0
 def get_closing(self, *, cycle: int) -> ReservationSet:
     closing = self.calendar.get_closing(cycle=cycle)
     result = ReservationSet()
     for reservation in closing.values():
         if not reservation.is_failed():
             self.calendar.add_pending(reservation=reservation)
             result.add(reservation=reservation)
         else:
             self.logger.warning(
                 "Removing failed reservation from the closing list: {}".
                 format(reservation))
     return result
 def get_all_reservations(self, *, cycle: int) -> ReservationSet:
     """
     Returns all reservations associated with cycles up to and including the specified cycle
     @params cycle : cycle
     @returns a set of reservations associated with the cycles up to and including specified cycle.
     Note that removing from the set will not affect the ReservationList
     """
     result = ReservationSet()
     for entry in self.rset_wrapper_list:
         if entry.cycle <= cycle:
             for reservation in entry.reservation_set.values():
                 result.add(reservation=reservation)
         else:
             break
     return result
Esempio n. 4
0
    def get_reservations(self,
                         *,
                         time: int = None,
                         rtype: ResourceType = None) -> ReservationSet:
        """
        Performs an intersection query: returns all reservations from the
        specified resource type present in the collection that are active at the
        specified time instance.
        @params time : time instance
        @params rtype : resource type
        @returns reservations set containing active reservations
        """
        if time is None and rtype is None:
            return self.reservation_set

        result = ReservationSet()
        key = ReservationWrapper(reservation=None, start=time, end=time)

        # Find the location of key in the list.
        index = binary_search(a=self.list, x=key)
        if index < 0:
            index = -index - 1

        # Scan the upper part of the list. We need to scan the whole list.
        i = index
        count = self.size()
        while i < count:
            entry = self.list[i]

            if rtype is None or rtype == entry.reservation.getType():
                if entry.start <= time <= entry.end:
                    result.add(reservation=entry.reservation)
            i += 1

        # Scan the lower part of the list until no further intersections are possible
        i = index - 1
        while i >= 0:
            entry = self.list[i]
            if entry.end < time:
                break

            if entry.start <= time and rtype is None or entry.reservation.getType(
            ) == rtype:
                result.add(reservation=entry.reservation)
            i -= 1

        return result
Esempio n. 5
0
    def test_pending(self):
        cal = self._get_calendar()
        rset = ReservationSet()

        for i in range(5):
            r = self._make_reservation(id=str(i))
            rset.add(reservation=r)
            cal.add_pending(reservation=r)
            temp = cal.get_pending()
            self.check_set(rset=rset, check=temp)
            temp.remove(reservation=r)
            temp = cal.get_pending()
            self.check_set(rset=rset, check=temp)

        for i in range(5):
            r = self._make_reservation(id=str(i))
            rset.remove(reservation=r)
            cal.remove_pending(reservation=r)
            temp = cal.get_pending()
            self.check_set(rset=rset, check=temp)
Esempio n. 6
0
    def process_renewing(self, *, renewing: ReservationSet) -> ReservationSet:
        """
        Performs checks on renewing reservations. Updates the terms to
        suggest new terms, stores the extend on the pending list. Returns a
        fresh ReservationSet of expiring reservations to try to renew in this
        bidding cycle.

        @param renewing collection of the renewing reservations

        @return non-null set of renewals
        """
        result = ReservationSet()
        if renewing is None:
            return None

        #self.logger.debug("Expiring = {}".format(renewing.size()))

        for reservation in renewing.values():
            self.logger.debug("Expiring res: {}".format(reservation))

            if reservation.is_renewable():
                self.logger.debug("This is a renewable expiring reservation")

                term = reservation.get_term()

                term = term.extend()

                reservation.set_approved(term=term,
                                         approved_resources=reservation.
                                         get_resources().abstract_clone())

                result.add(reservation=reservation)
                self.calendar.add_pending(reservation=reservation)
            else:
                self.logger.debug(
                    "This is not a renewable expiring reservation")

        return result
    def add_reservation(self, *, reservation: ABCReservationMixin, cycle: int):
        """
        Adds a reservation associated with a given cycle.
        @params reservation:  the reservation
        @params cycle: the cycle with which to associate the reservation
        """
        if reservation is None or cycle < 0 or reservation.get_reservation_id(
        ) is None:
            raise FrameworkException(Constants.INVALID_ARGUMENT)

        if reservation.get_reservation_id() in self.reservation_id_to_cycle:
            existing_cycle = self.reservation_id_to_cycle[
                reservation.get_reservation_id()]
            if existing_cycle == cycle:
                return
            else:
                raise RuntimeError(
                    "Reservation: #{} is already in the list at a different cycle. Please remove it first, "
                    "before adding to a different cycle".format(
                        reservation.get_reservation_id()))

        reservation_set = None
        if cycle in self.cycle_to_rset:
            reservation_set = self.cycle_to_rset[cycle]
        else:
            reservation_set = ReservationSet()

        if reservation_set.contains(reservation=reservation) is False:
            reservation_set.add(reservation=reservation)
            self.count += 1

        if cycle not in self.cycle_to_rset:
            self.add_to_list(reservation_set=reservation_set, cycle=cycle)
            self.cycle_to_rset[cycle] = reservation_set

        self.reservation_id_to_cycle[reservation.get_reservation_id()] = cycle
Esempio n. 8
0
class Authority(ActorMixin, ABCAuthority):
    """
    Authority is the base implementation for a site authority actor.
    """
    def __init__(self, *, identity: AuthToken = None, clock: ActorClock = None):
        super().__init__(auth=identity, clock=clock)
        self.type = ActorType.Authority
        # Initialization status.
        self.initialized = False
        # Reservations to redeem once the actor recovers.
        self.redeeming = ReservationSet()
        # Reservations to extendLease for once the actor recovers.
        self.extending_lease = ReservationSet()
        # Reservations to modifyLease for once the actor recovers
        self.modifying_lease = ReservationSet()

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['recovered']
        del state['wrapper']
        del state['logger']
        del state['clock']
        del state['current_cycle']
        del state['first_tick']
        del state['stopped']
        del state['initialized']
        del state['thread_lock']
        del state['thread']
        del state['timer_queue']
        del state['event_queue']
        del state['reservation_tracker']
        del state['subscription_id']
        del state['actor_main_lock']
        del state['closing']
        del state['message_service']

        del state['redeeming']
        del state['extending_lease']
        del state['modifying_lease']

        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.recovered = False
        self.wrapper = None
        self.logger = None
        self.clock = None
        self.current_cycle = -1
        self.first_tick = True
        self.stopped = False
        self.initialized = False
        self.thread = None
        self.thread_lock = threading.Lock()
        self.timer_queue = queue.Queue()
        self.event_queue = queue.Queue()
        self.reservation_tracker = None
        self.subscription_id = None
        self.actor_main_lock = threading.Condition()
        self.closing = ReservationSet()
        self.message_service = None

        self.redeeming = ReservationSet()
        self.extending_lease = ReservationSet()
        self.modifying_lease = ReservationSet()

    def register_client_slice(self, *, slice_obj: ABCSlice):
        self.wrapper.register_slice(slice_object=slice_obj)

    def claim_delegation(self, *, delegation: ABCDelegation, callback: ABCClientCallbackProxy, caller: AuthToken,
                         id_token: str = None):
        slice_obj = delegation.get_slice_object()
        if slice_obj is not None:
            slice_obj.set_broker_client()

        self.wrapper.claim_delegation_request(delegation=delegation, caller=caller, callback=callback,
                                              id_token=id_token)

    def reclaim_delegation(self, *, delegation: ABCDelegation, callback: ABCClientCallbackProxy, caller: AuthToken,
                           id_token: str = None):

        slice_obj = delegation.get_slice_object()
        if slice_obj is not None:
            slice_obj.set_broker_client()

        self.wrapper.reclaim_delegation_request(delegation=delegation, caller=caller, callback=callback,
                                                id_token=id_token)

    def close_by_caller(self, *, reservation: ABCReservationMixin, caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise AuthorityException(Constants.INVALID_ACTOR_STATE)

        self.wrapper.close_request(reservation=reservation, caller=caller, compare_sequence_numbers=True)

    def close_expiring(self, *, cycle: int):
        """
        Closes the expiring reservations for the specified cycle.

        @param cycle
                   cycle number
        """
        expired = self.policy.get_closing(cycle=cycle)
        if expired is not None:
            # self.logger.info("Authority expiring for cycle {} = {}".format(cycle, expired))
            self.close_reservations(reservations=expired)

    def donate_delegation(self, *, delegation: ABCDelegation):
        self.policy.donate_delegation(delegation=delegation)

    def eject(self, *, resources: ResourceSet):
        self.policy.eject(resources=resources)

    def advertise(self, *, delegation: ABCPropertyGraph, delegation_name: str, client: AuthToken) -> str:
        slice_obj = SliceFactory.create(slice_id=ID(), name=client.get_name())
        slice_obj.set_owner(owner=client)
        slice_obj.set_broker_client()

        dlg_obj = DelegationFactory.create(did=delegation.get_graph_id(), slice_id=slice_obj.get_slice_id(),
                                           delegation_name=delegation_name)
        dlg_obj.set_slice_object(slice_object=slice_obj)
        dlg_obj.set_graph(graph=delegation)
        self.wrapper.advertise(delegation=dlg_obj, client=client)
        return dlg_obj.get_delegation_id()

    def extend_lease(self, *, reservation: ABCAuthorityReservation, caller: AuthToken = None):
        if caller is None:
            if not self.recovered:
                self.extending_lease.add(reservation=reservation)
            else:
                self.wrapper.extend_lease_request(reservation=reservation, caller=reservation.get_client_auth_token(),
                                                  compare_sequence_numbers=False)
        else:
            if not self.is_recovered() or self.is_stopped():
                raise AuthorityException(Constants.INVALID_ACTOR_STATE)
            self.wrapper.extend_lease_request(reservation=reservation, caller=caller, compare_sequence_numbers=True)

    def modify_lease(self, *, reservation: ABCAuthorityReservation, caller: AuthToken):
        if caller is None:
            if not self.recovered:
                self.modifying_lease.add(reservation=reservation)
            else:
                self.wrapper.modify_lease_request(reservation=reservation, caller=reservation.get_client_auth_token(),
                                                  compare_sequence_numbers=False)
        else:
            if not self.is_recovered() or self.stopped:
                raise AuthorityException(Constants.INVALID_ACTOR_STATE)
            self.wrapper.modify_lease_request(reservation=reservation, caller=caller, compare_sequence_numbers=True)

    def extend_ticket(self, *, reservation: ABCReservationMixin, caller: AuthToken):
        slice_obj = reservation.get_slice()
        if slice_obj is not None:
            slice_obj.set_broker_client()

        self.wrapper.extend_ticket_request(reservation=reservation, caller=caller, compare_sequence_numbers=True)

    def relinquish(self, *, reservation: ABCReservationMixin, caller: AuthToken):
        if not self.is_recovered() or self.stopped:
            raise AuthorityException(Constants.INVALID_ACTOR_STATE)
        self.wrapper.relinquish_request(reservation=reservation, caller=caller)

    def freed(self, *, resources: ResourceSet):
        self.policy.freed(resources=resources)

    def redeem(self, *, reservation: ABCReservationMixin, callback: ABCControllerCallbackProxy = None,
               caller: AuthToken = None):
        if callback is None and caller is None:
            if not self.recovered:
                self.redeeming.add(reservation=reservation)
            else:
                self.wrapper.redeem_request(reservation=reservation, caller=reservation.get_client_auth_token(),
                                            callback=reservation.get_callback(), compare_sequence_numbers=False)
        else:
            if not self.is_recovered() or self.is_stopped():
                raise AuthorityException(Constants.INVALID_ACTOR_STATE)

            if self.plugin.validate_incoming(reservation=reservation, auth=caller):
                self.wrapper.redeem_request(reservation=reservation, caller=caller, callback=callback,
                                            compare_sequence_numbers=True)
            else:
                self.logger.error("the redeem request is invalid")
        self.logger.debug("Completed processing Redeem Request")

    def ticket(self, *, reservation: ABCReservationMixin, callback: ABCClientCallbackProxy, caller: AuthToken):
        slice_obj = reservation.get_slice()
        if slice_obj is not None:
            slice_obj.set_broker_client()

        self.wrapper.ticket_request(reservation=reservation, caller=caller, callback=callback, compare_seq_numbers=True)

    def tick_handler(self):
        # close expired reservations
        self.close_expiring(cycle=self.current_cycle)
        # process all requests for the current cycle
        self.policy.assign(cycle=self.current_cycle)

    def register_client(self, *, client: Client):
        db = self.plugin.get_database()

        try:
            db.get_client(guid=client.get_guid())
        except Exception as e:
            self.logger.debug("Client does not exist e:{}".format(e))

        db.add_client(client=client)

    def unregister_client(self, *, guid: ID):
        db = self.plugin.get_database()
        db.remove_client(guid=guid)

    def get_client(self, *, guid: ID) -> Client:
        db = self.plugin.get_database()

        return db.get_client(guid=guid)

    def redeem_reservations(self, *, rset: ReservationSet):
        """
        Redeem all reservations:
        @param rset: reservation set
        """
        for reservation in rset.values():
            try:
                if isinstance(reservation, ABCAuthorityReservation):
                    self.redeem(reservation=reservation)
                else:
                    self.logger.warning("Reservation # {} cannot be redeemed".format(reservation.get_reservation_id()))
            except Exception as e:
                self.logger.error("Could not redeem for # {} {}".format(reservation.get_reservation_id(), e))

    def extend_lease_reservations(self, *, rset: ReservationSet):
        """
        Extend all reservations:
        @param rset: reservation set
        """
        for reservation in rset.values():
            try:
                self.extend_lease(reservation=reservation)
            except Exception as e:
                self.logger.error("Could not redeem for # {} {}".format(reservation.get_reservation_id(), e))

    def issue_delayed(self):
        super().issue_delayed()
        self.redeem_reservations(rset=self.redeeming)
        self.redeeming.clear()
        self.extend_lease_reservations(rset=self.extending_lease)
        self.extending_lease.clear()

    @staticmethod
    def get_management_object_class() -> str:
        return AuthorityManagementObject.__name__

    @staticmethod
    def get_management_object_module() -> str:
        return AuthorityManagementObject.__module__

    @staticmethod
    def get_kafka_service_class() -> str:
        return AuthorityService.__name__

    @staticmethod
    def get_kafka_service_module() -> str:
        return AuthorityService.__module__

    @staticmethod
    def get_mgmt_kafka_service_class() -> str:
        return KafkaAuthorityService.__name__

    @staticmethod
    def get_mgmt_kafka_service_module() -> str:
        return KafkaAuthorityService.__module__

    def set_aggregate_resource_model(self, aggregate_resource_model: Neo4jARMGraph):
        """
        Set aggregate resource model
        :param aggregate_resource_model: resource model
        :return:
        """
        self.policy.set_aggregate_resource_model(aggregate_resource_model=aggregate_resource_model)

    def load_model(self, *, graph_id: str):
        self.policy.set_aggregate_resource_model_graph_id(graph_id=graph_id)
        self.policy.load_aggregate_resource_model()
Esempio n. 9
0
class ReservationHoldings:
    """
    This class maintains a collection of reservations. Each reservation is
    associated with a validity interval. The class allows to answer intersection
    queries: what reservations are valid at a given time instance.

    As time goes by, the class can be purged from irrelevant reservation records
    by invoking tick(). Purging is strongly recommended as it reduces the cost of intersection queries.

    An attempt has been made to optimize the cost of using this data structure.
    Inserts are O(log(n)). Queries, however, may take between O(log(n)) and O(n).
    """
    def __init__(self):
        # List of reservation wrappers sorted by increasing end time.
        self.list = []
        # All reservations stored in this collection.
        self.reservation_set = ReservationSet()
        # Map of reservations to ReservationWrappers. Needed when removing a reservation.
        self.map = {}

    def add_reservation(self, *, reservation: ABCReservationMixin, start: int,
                        end: int):
        """
        Adds a reservation to the collection for the specified period of time.
        The interval is closed on both sides.
        @params reservation : reservation to add
        @params start : start time
        @params end : end time
        """
        # If this is an extended reservation, we may already have it in the
        # list (with potentially different start and end times). Remove the
        # previous entry if this is the case.
        my_start = start
        entry = None
        if reservation.get_reservation_id() in self.map:
            entry = self.map[reservation.get_reservation_id()]
            if entry is not None:
                assert (start - entry.end) <= 1
                my_start = entry.start
                self.remove_reservation(reservation=reservation)

        entry = ReservationWrapper(reservation=reservation,
                                   start=my_start,
                                   end=end)
        self.add_to_list(entry=entry)
        self.reservation_set.add(reservation=reservation)
        self.map[reservation.get_reservation_id()] = entry

    def add_to_list(self, *, entry: ReservationWrapper):
        """
        Adds the entry to the linked list. Maintains the list in sorted order.
        Cost: O(log(n)).
        @params entry : entry to add
        """
        bisect.insort_left(self.list, entry)

    def clear(self):
        """
        Clears the collection.
        """
        self.map.clear()
        self.list.clear()
        self.reservation_set.clear()

    def get_reservations(self,
                         *,
                         time: int = None,
                         rtype: ResourceType = None) -> ReservationSet:
        """
        Performs an intersection query: returns all reservations from the
        specified resource type present in the collection that are active at the
        specified time instance.
        @params time : time instance
        @params rtype : resource type
        @returns reservations set containing active reservations
        """
        if time is None and rtype is None:
            return self.reservation_set

        result = ReservationSet()
        key = ReservationWrapper(reservation=None, start=time, end=time)

        # Find the location of key in the list.
        index = binary_search(a=self.list, x=key)
        if index < 0:
            index = -index - 1

        # Scan the upper part of the list. We need to scan the whole list.
        i = index
        count = self.size()
        while i < count:
            entry = self.list[i]

            if rtype is None or rtype == entry.reservation.getType():
                if entry.start <= time <= entry.end:
                    result.add(reservation=entry.reservation)
            i += 1

        # Scan the lower part of the list until no further intersections are possible
        i = index - 1
        while i >= 0:
            entry = self.list[i]
            if entry.end < time:
                break

            if entry.start <= time and rtype is None or entry.reservation.getType(
            ) == rtype:
                result.add(reservation=entry.reservation)
            i -= 1

        return result

    def remove_from_list(self, *, entry: ReservationWrapper):
        """
        Removes a reservation from the collection.
        @params reservation : reservation to remove
        """
        index = binary_search(a=self.list, x=entry)

        if index >= 0:
            self.list.pop(index)

    def remove_reservation(self, *, reservation: ABCReservationMixin):
        """
        Removes a reservation from the collection.
        @params reservation : reservation to remove
        """
        if reservation.get_reservation_id() in self.map:
            entry = self.map[reservation.get_reservation_id()]
            self.map.pop(reservation.get_reservation_id())
            self.reservation_set.remove(reservation=reservation)
            self.remove_from_list(entry=entry)

    def size(self) -> int:
        """
        Returns the size of the collection.
        @returns size of the collection
        """
        return self.reservation_set.size()

    def tick(self, *, time: int):
        """
        Removes all reservations that have end time not after the given cycle.
        @params time : time
        """
        while True:
            if len(self.list) > 0:
                entry = self.list[0]
                if entry.end <= time:
                    self.list.remove(entry)
                    self.reservation_set.remove(reservation=entry.reservation)
                    self.map.pop(entry.reservation.get_reservation_id())
                else:
                    break
            else:
                break
Esempio n. 10
0
class ControllerTicketReviewPolicy(ControllerSimplePolicy):
    """
    This implementation of a Controller policy is almost identical to the parent ControllerSimplePolicy.
    The only real difference is that it addresses that Tickets should not be redeemed if any reservations are
    currently Failed or Nascent.  This effectively acts as a "gate" between the Controller and AM.
    All reservations must be Ticketed, before any reservations are allowed to be redeemed.
    """
    def __init__(self):
        super().__init__()
        self.pending_redeem = ReservationSet()

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['logger']
        del state['actor']
        del state['clock']
        del state['initialized']
        del state['pending_notify']
        del state['lazy_close']
        del state['pending_redeem']

        return state

    def __setstate__(self, state):
        self.__dict__.update(state)

        self.logger = None
        self.actor = None
        self.clock = None
        self.initialized = False
        self.pending_notify = ReservationSet()
        self.lazy_close = False
        self.pending_redeem = ReservationSet()

    def check_pending(self):
        """
        Check to make sure all reservations are Ticketed (not Failed or Nascent)
        before calling the parent method.

        @throws Exception in case of error
        """
        # add all of our pendingRedeem, so they can be checked
        for reservation in self.pending_redeem.values():
            self.calendar.add_pending(reservation=reservation)

        # get set of reservations that need to be redeemed
        my_pending = self.calendar.get_pending()

        # keep track of status of the slice containing each reservation
        slice_status_map = {}

        # nothing to do!
        if my_pending is None:
            return

        # check the status of the Slice of each reservation
        for reservation in my_pending.values():
            slice_obj = reservation.get_slice()
            slice_id = slice_obj.get_slice_id()

            # only want to do this for 'new' tickets
            if reservation.is_failed() or reservation.is_ticketed():
                # check if we've examined this slice already
                if slice_id not in slice_status_map:
                    # set the default status
                    slice_status_map[
                        slice_id] = TicketReviewSliceState.Redeemable
                    # examine every reservation contained within the slice,
                    # looking for either a Failed or Nascent reservation
                    # we have to look at everything in a slice once, to determine all/any Sites with failures
                    for slice_reservation in slice_obj.get_reservations(
                    ).values():

                        # If any Reservations that are being redeemed, that means the
                        # slice has already cleared TicketReview.
                        if slice_reservation.is_redeeming():
                            if slice_status_map[
                                    slice_id] == TicketReviewSliceState.Nascent:
                                # There shouldn't be any Nascent reservations, if a reservation is being Redeemed.
                                self.logger.error(
                                    "Nascent reservation found while Reservation {} in slice {} is redeeming"
                                    .format(
                                        slice_reservation.get_reservation_id(),
                                        slice_obj.get_name()))

                            # We may have previously found a Failed Reservation,
                            # but if a ticketed reservation is being redeemed,
                            # the failure _should_ be from the AM, not Controller
                            # so it should be ignored by TicketReview
                            slice_status_map[
                                slice_id] = TicketReviewSliceState.Redeemable

                            #  we don't need to look at any other reservations in this slice
                            break

                        # if any tickets are Nascent,
                        # as soon as we remove the Failed reservation,
                        # those Nascent tickets might get redeemed.
                        # we must wait to Close any failed reservations
                        # until all Nascent tickets are either Ticketed or Failed
                        if slice_reservation.is_nascent():
                            self.logger.debug(
                                "Found Nascent Reservation {} in slice {} when check_pending for {}"
                                .format(slice_reservation.get_reservation_id(),
                                        slice_obj.get_name(),
                                        reservation.get_reservation_id()))

                            slice_status_map[
                                slice_id] = TicketReviewSliceState.Nascent
                            # once we have found a Nascent reservation, that is what we treat the entire slice
                            break

                        # track Failed reservations, but need to keep looking for Nascent or Redeemable.
                        if slice_reservation.is_failed():
                            self.logger.debug(
                                "Found failed reservation {} in slice {} when check_pending for {}"
                                .format(slice_reservation.get_reservation_id(),
                                        slice_obj.get_name(),
                                        reservation.get_reservation_id()))
                            slice_status_map[
                                slice_id] = TicketReviewSliceState.Failing

                # take action on the current reservation
                if slice_status_map[
                        slice_id] == TicketReviewSliceState.Failing:
                    if reservation.get_resources(
                    ) is not None and reservation.get_resources().get_type(
                    ) is not None:
                        msg = f"TicketReviewPolicy: Closing reservation {reservation.get_reservation_id()} due to " \
                              f"failure in slice {slice_obj.get_name()}"
                        self.logger.info(msg)

                        if not reservation.is_failed():
                            update_data = UpdateData()
                            update_data.failed = True
                            update_data.message = Constants.CLOSURE_BY_TICKET_REVIEW_POLICY
                            reservation.mark_close_by_ticket_review(
                                update_data=update_data)
                        self.actor.close(reservation=reservation)
                        self.calendar.remove_pending(reservation=reservation)
                        self.pending_notify.remove(reservation=reservation)

                elif slice_status_map[
                        slice_id] == TicketReviewSliceState.Nascent:
                    self.logger.info(
                        "Moving reservation {} to pending redeem list due to nascent reservation in slice {}"
                        .format(reservation.get_reservation_id(),
                                slice_obj.get_name()))
                    self.pending_redeem.add(reservation=reservation)
                    self.calendar.remove_pending(reservation=reservation)
                else:
                    # we don't need to look at any other reservations in this slice
                    self.logger.debug(
                        "Removing from pendingRedeem: {}".format(reservation))
                    self.pending_redeem.remove(reservation=reservation)
            else:
                # Remove active or close reservations
                self.logger.debug(
                    "Removing from pendingRedeem: {}".format(reservation))
                self.pending_redeem.remove(reservation=reservation)

        super().check_pending()
Esempio n. 11
0
class Broker(ActorMixin, ABCBrokerMixin):
    """
    Broker offers the base for all broker actors.
    """
    def __init__(self, *, identity: AuthToken = None, clock: ActorClock = None):
        super().__init__(auth=identity, clock=clock)
        # Recovered reservations that need to obtain tickets (both server and client roles).
        self.ticketing = ReservationSet()
        # Recovered reservations that need to extend tickets (both server and client roles).
        self.extending = ReservationSet()
        # The peer registry.
        self.registry = PeerRegistry()
        # Initialization status.
        self.initialized = False
        self.type = ActorType.Broker

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['recovered']
        del state['wrapper']
        del state['logger']
        del state['clock']
        del state['current_cycle']
        del state['first_tick']
        del state['stopped']
        del state['initialized']
        del state['thread_lock']
        del state['thread']
        del state['timer_queue']
        del state['event_queue']
        del state['reservation_tracker']
        del state['subscription_id']
        del state['actor_main_lock']
        del state['closing']
        del state['message_service']

        del state['ticketing']
        del state['extending']
        del state['registry']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)

        self.recovered = False
        self.wrapper = None
        self.logger = None
        self.clock = None
        self.current_cycle = -1
        self.first_tick = True
        self.stopped = False
        self.initialized = False
        self.thread = None
        self.thread_lock = threading.Lock()
        self.timer_queue = queue.Queue()
        self.event_queue = queue.Queue()
        self.reservation_tracker = None
        self.subscription_id = None
        self.actor_main_lock = threading.Condition()
        self.closing = ReservationSet()
        self.message_service = None

        self.ticketing = ReservationSet()
        self.extending = ReservationSet()
        self.registry = PeerRegistry()

    def actor_added(self):
        super().actor_added()
        self.registry.actor_added()

    def add_broker(self, *, broker: ABCBrokerProxy):
        self.registry.add_broker(broker=broker)

    def register_client_slice(self, *, slice_obj: ABCSlice):
        self.wrapper.register_slice(slice_object=slice_obj)

    def donate_delegation(self, *, delegation: ABCDelegation):
        ## This function is used by AUTs
        self.policy.donate_delegation(delegation=delegation)

    def bid(self, *, cycle: int):
        """
        Bids for resources as dictated by the bidding policy for the current cycle.

        @param cycle cycle

        @throws Exception in case of error
        """
        candidates = None
        candidates = self.policy.formulate_bids(cycle=cycle)
        if candidates is not None:
            for reservation in candidates.get_ticketing().values():
                try:
                    self.wrapper.ticket(reservation=reservation, destination=self)
                except Exception as e:
                    self.logger.error("unexpected ticket failure for #{} e: {}".format(
                        reservation.get_reservation_id(), e))

            for reservation in candidates.get_extending().values():
                try:
                    self.wrapper.extend_ticket(reservation=reservation)
                except Exception as e:
                    self.logger.error("unexpected extend failure for #{}".format(reservation.get_reservation_id()))

    def claim_delegation_client(self, *, delegation_id: str = None, slice_object: ABCSlice = None,
                                broker: ABCBrokerProxy = None, id_token: str = None) -> ABCDelegation:
        if delegation_id is None:
            raise BrokerException(error_code=ExceptionErrorCode.INVALID_ARGUMENT, msg="delegation_id")

        if broker is None:
            broker = self.get_default_broker()

        if slice_object is None:
            slice_object = self.get_default_slice()
            if slice_object is None:
                slice_object = SliceFactory.create(slice_id=ID(), name=self.identity.get_name())
                slice_object.set_owner(owner=self.identity)
                slice_object.set_inventory(value=True)

        delegation = BrokerDelegationFactory.create(did=delegation_id, slice_id=slice_object.get_slice_id(),
                                                    broker=broker)
        delegation.set_exported(value=True)
        delegation.set_slice_object(slice_object=slice_object)

        self.wrapper.delegate(delegation=delegation, destination=self, id_token=id_token)
        return delegation

    def reclaim_delegation_client(self, *, delegation_id: str = None, slice_object: ABCSlice = None,
                                  broker: ABCBrokerProxy = None, id_token: str = None) -> ABCDelegation:
        if delegation_id is None:
            raise BrokerException(error_code=ExceptionErrorCode.INVALID_ARGUMENT, msg="delegation_id")

        if broker is None:
            broker = self.get_default_broker()

        if slice_object is None:
            slice_object = self.get_default_slice()
            if slice_object is None:
                slice_object = SliceFactory.create(slice_id=ID(), name=self.identity.get_name())
                slice_object.set_owner(owner=self.identity)
                slice_object.set_inventory(value=True)

        delegation = BrokerDelegationFactory.create(did=delegation_id, slice_id=slice_object.get_slice_id(),
                                                    broker=broker)
        delegation.set_slice_object(slice_object=slice_object)
        delegation.set_exported(value=True)

        callback = ActorRegistrySingleton.get().get_callback(protocol=Constants.PROTOCOL_KAFKA,
                                                             actor_name=self.get_name())
        if callback is None:
            raise BrokerException(error_code=ExceptionErrorCode.NOT_SUPPORTED, msg="callback is None")

        delegation.prepare(callback=callback, logger=self.logger)
        delegation.validate_outgoing()

        self.wrapper.reclaim_delegation_request(delegation=delegation, caller=broker, callback=callback,
                                                id_token=id_token)

        return delegation

    def claim_delegation(self, *, delegation: ABCDelegation, callback: ABCClientCallbackProxy, caller: AuthToken,
                         id_token: str = None):
        if not self.is_recovered() or self.is_stopped():
            raise BrokerException(error_code=ExceptionErrorCode.UNEXPECTED_STATE, msg="of actor")
        self.wrapper.claim_delegation_request(delegation=delegation, caller=caller, callback=callback,
                                              id_token=id_token)

    def reclaim_delegation(self, *, delegation: ABCDelegation, callback: ABCClientCallbackProxy, caller: AuthToken,
                           id_token: str = None):
        if not self.is_recovered() or self.is_stopped():
            raise BrokerException(error_code=ExceptionErrorCode.UNEXPECTED_STATE, msg="of actor")
        self.wrapper.reclaim_delegation_request(delegation=delegation, caller=caller, callback=callback,
                                                id_token=id_token)

    def close_expiring(self, *, cycle: int):
        """
        Closes all expiring reservations.
        @param cycle cycle
        """
        expired = self.policy.get_closing(cycle=cycle)
        if expired is not None:
            self.logger.info("Broker expiring for cycle {} = {}".format(cycle, expired))
            self.close_reservations(reservations=expired)

    def demand(self, *, rid: ID):
        if rid is None:
            raise BrokerException(error_code=ExceptionErrorCode.INVALID_ARGUMENT, msg="rid")
        reservation = self.get_reservation(rid=rid)
        if reservation is None:
            raise BrokerException(error_code=ExceptionErrorCode.NOT_FOUND, msg=f"rid={rid}")

        self.policy.demand(reservation=reservation)
        reservation.set_policy(policy=self.policy)

    def advertise(self, *, delegation: ABCPropertyGraph, delegation_name: str, client: AuthToken) -> ID:
        slice_obj = SliceFactory.create(slice_id=ID(), name=client.get_name())
        slice_obj.set_owner(owner=client)
        slice_obj.set_broker_client()

        dlg_obj = DelegationFactory.create(did=delegation.get_graph_id(), slice_id=slice_obj.get_slice_id(),
                                           delegation_name=delegation_name)
        dlg_obj.set_slice_object(slice_object=slice_obj)
        dlg_obj.set_graph(graph=delegation)
        self.wrapper.advertise(delegation=dlg_obj, client=client)
        return dlg_obj.get_delegation_id()

    def extend_ticket_client(self, *, reservation: ABCClientReservation):
        if not self.recovered:
            self.extending.add(reservation=reservation)
        else:
            self.wrapper.extend_ticket(reservation=reservation)

    def extend_tickets_client(self, *, rset: ReservationSet):
        for reservation in rset.values():
            try:
                if isinstance(reservation, ABCBrokerReservation):
                    self.extend_ticket_broker(reservation=reservation)
                elif isinstance(reservation, ABCClientReservation):
                    self.extend_ticket_client(reservation=reservation)
                else:
                    self.logger.warning("Reservation #{} cannot be ticketed".format(reservation.get_reservation_id()))
            except Exception as e:
                self.logger.error("Could not ticket for # {} e: {}".format(reservation.get_reservation_id(), e))

    def extend_ticket_broker(self, *, reservation: ABCBrokerReservation):
        if not self.recovered:
            self.extending.add(reservation=reservation)
        else:
            self.wrapper.extend_ticket_request(reservation=reservation, caller=reservation.get_client_auth_token(),
                                               compare_sequence_numbers=False)

    def extend_ticket(self, *, reservation: ABCReservationMixin, caller: AuthToken):
        if not self.recovered or self.is_stopped():
            raise BrokerException(error_code=ExceptionErrorCode.UNEXPECTED_STATE, msg="of actor")
        self.wrapper.extend_ticket_request(reservation=reservation, caller=caller, compare_sequence_numbers=True)

    def get_broker(self, *, guid: ID) -> ABCBrokerProxy:
        return self.registry.get_broker(guid=guid)

    def get_brokers(self) -> list:
        return self.registry.get_brokers()

    def get_default_broker(self) -> ABCBrokerProxy:
        return self.registry.get_default_broker()

    def get_default_slice(self) -> ABCSlice:
        """
        Get default inventory slice for the broker
        @return inventory slice for broker
        """
        slice_list = self.get_inventory_slices()
        if slice_list is not None and len(slice_list) > 0:
            return slice_list[0]
        return None

    def initialize(self):
        if not self.initialized:
            super().initialize()
            self.registry.set_slices_plugin(plugin=self.plugin)
            self.registry.initialize()
            self.initialized = True

    def issue_delayed(self):
        super().issue_delayed()

        self.extend_tickets_client(rset=self.extending)
        self.extending.clear()

        self.tickets_client(rset=self.ticketing)
        self.ticketing.clear()

    def ticket_client(self, *, reservation: ABCClientReservation):
        if not self.recovered:
            self.ticketing.add(reservation=reservation)
        else:
            self.wrapper.ticket(reservation=reservation, destination=self)

    def tickets_client(self, *, rset: ReservationSet):
        for reservation in rset.values():
            try:
                if isinstance(reservation, ABCBrokerReservation):
                    self.ticket_broker(reservation=reservation)
                elif isinstance(reservation, ABCClientReservation):
                    self.ticket_client(reservation=reservation)
                else:
                    self.logger.warning("Reservation #{} cannot be ticketed".format(reservation.get_reservation_id()))
            except Exception as e:
                self.logger.error("Could not ticket for #{} e: {}".format(reservation.get_reservation_id(), e))

    def ticket_broker(self, *, reservation: ABCBrokerReservation):
        if not self.recovered:
            self.ticketing.add(reservation=reservation)
        else:
            self.wrapper.ticket_request(reservation=reservation, caller=reservation.get_client_auth_token(),
                                        callback=reservation.get_callback(), compare_seq_numbers=False)

    def ticket(self, *, reservation: ABCReservationMixin, callback: ABCClientCallbackProxy, caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise BrokerException(error_code=ExceptionErrorCode.UNEXPECTED_STATE, msg="of actor")

        self.wrapper.ticket_request(reservation=reservation, caller=caller, callback=callback, compare_seq_numbers=True)

    def relinquish(self, *, reservation: ABCReservationMixin, caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise BrokerException(error_code=ExceptionErrorCode.UNEXPECTED_STATE, msg="of actor")
        self.wrapper.relinquish_request(reservation=reservation, caller=caller)

    def tick_handler(self):
        self.policy.allocate(cycle=self.current_cycle)
        self.bid(cycle=self.current_cycle)
        self.close_expiring(cycle=self.current_cycle)

    def update_lease(self, *, reservation: ABCReservationMixin, update_data, caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise BrokerException("This actor cannot receive calls")

        self.wrapper.update_lease(reservation=reservation, update_data=update_data, caller=caller)

    def update_ticket(self, *, reservation: ABCReservationMixin, update_data, caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise BrokerException(error_code=ExceptionErrorCode.UNEXPECTED_STATE, msg="of actor")
        self.wrapper.update_ticket(reservation=reservation, update_data=update_data, caller=caller)

    def update_delegation(self, *, delegation: ABCDelegation, update_data, caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise BrokerException(error_code=ExceptionErrorCode.UNEXPECTED_STATE, msg="of actor")
        self.wrapper.update_delegation(delegation=delegation, update_data=update_data, caller=caller)

    def register_client(self, *, client: Client):
        database = self.plugin.get_database()

        try:
            database.get_client(guid=client.get_guid())
        except Exception as e:
            raise BrokerException(error_code=ExceptionErrorCode.NOT_FOUND, msg=f"client: {client.get_guid()} e: {e}")

        try:
            database.add_client(client=client)
        except Exception as e:
            raise BrokerException(error_code=ExceptionErrorCode.FAILURE, msg=f"client: {client.get_guid()} e: {e}")

    def unregister_client(self, *, guid: ID):
        database = self.plugin.get_database()

        try:
            database.get_client(guid=guid)
        except Exception as e:
            raise BrokerException(error_code=ExceptionErrorCode.NOT_FOUND, msg=f"client: {guid} e: {e}")

        try:
            database.remove_client(guid=guid)
        except Exception as e:
            raise BrokerException(error_code=ExceptionErrorCode.FAILURE, msg=f"client: {guid} e: {e}")

    def get_client(self, *, guid: ID) -> Client:
        database = self.plugin.get_database()
        ret_val = None
        try:
            ret_val = database.get_client(guid=guid)
        except Exception as e:
            raise BrokerException(error_code=ExceptionErrorCode.NOT_FOUND, msg=f"client: {guid} e: {e}")

        return ret_val

    def modify(self, *, reservation_id: ID, modify_properties: dict):
        return

    @staticmethod
    def get_management_object_class() -> str:
        return BrokerManagementObject.__name__

    @staticmethod
    def get_management_object_module() -> str:
        return BrokerManagementObject.__module__

    @staticmethod
    def get_kafka_service_class() -> str:
        return BrokerService.__name__

    @staticmethod
    def get_kafka_service_module() -> str:
        return BrokerService.__module__

    @staticmethod
    def get_mgmt_kafka_service_class() -> str:
        return KafkaBrokerService.__name__

    @staticmethod
    def get_mgmt_kafka_service_module() -> str:
        return KafkaBrokerService.__module__
class ClientCalendar(BaseCalendar):
    """
    This a client-side calendar to be used by brokers  or service managers. A client
    calendar maintains the following lists:
    - demand: a list of reservations representing demand for resources
    - pending: a list of reservations with pending operations
    - renewing: a list of reservations organized by the time they must be renewed (cycle)
    - holdings: a list of active/granted reservations associated with their lease term.
      This list should be maintained using real time, not cycles.

    The renewing and holding lists are automatically purged by the implementation as
    time advances. The demand and pending list, however, must be purged manually by the
    user of this class.
    """
    def __init__(self, *, clock: ActorClock):
        """
        Constructor
        @params clock: clock factory
        """
        super().__init__(clock=clock)
        # Set of reservations representing the current demand. Callers are
        # responsible for removing serviced reservations
        self.demand = ReservationSet()
        # Set of reservations for which a request has been issued but no
        # confirmation has been received. Callers are responsible for removing
        # acknowledged reservations.
        self.pending = ReservationSet()
        # Set of reservations grouped by renewing time.
        self.renewing = ReservationList()
        # Set of active reservations.
        self.holdings = ReservationHoldings()
        self.lock = threading.Lock()

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['lock']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.lock = threading.Lock()

    def remove(self, *, reservation: ABCReservationMixin):
        """
        Removes the specified reservation from all internal calendar data structures
        @params reservation: reservation to remove
        """
        self.remove_demand(reservation=reservation)
        self.remove_pending(reservation=reservation)
        self.remove_renewing(reservation=reservation)
        self.remove_holdings(reservation=reservation)

    def remove_scheduled_or_in_progress(self, *,
                                        reservation: ABCReservationMixin):
        """
        Removes the specified reservations from all internal calendar data
        structures that represent operations to be scheduled in the future or
        operations that are currently in progress. Does not remove the
        reservation from the holdings list.
        @params reservation : reservation to remove
        """
        self.remove_demand(reservation=reservation)
        self.remove_pending(reservation=reservation)
        self.remove_renewing(reservation=reservation)

    def get_demand(self) -> ReservationSet:
        """
        Returns the known demand. Can be for resources starting at different times.
        @returns the set of demanded reservations.
        """
        try:
            self.lock.acquire()
            return self.demand.clone()
        finally:
            self.lock.release()

    def add_demand(self, *, reservation: ABCReservationMixin):
        """
        Adds a reservation to the demand list.
        @params reservation: reservation to add
        """
        try:
            self.lock.acquire()
            self.demand.add(reservation=reservation)
        finally:
            self.lock.release()

    def remove_demand(self, *, reservation: ABCReservationMixin):
        """
        Removes a reservation to the demand list.
        @params reservation: reservation to remove
        """
        try:
            self.lock.acquire()
            self.demand.remove(reservation=reservation)
        finally:
            self.lock.release()

    def get_pending(self) -> ReservationSet:
        """
        Returns the known pending. Can be for resources starting at different times.
        @returns the set of pending reservations.
        """
        try:
            self.lock.acquire()
            return self.pending.clone()
        finally:
            self.lock.release()

    def add_pending(self, *, reservation: ABCReservationMixin):
        """
        Adds a reservation to the pending list.
        @params reservation: reservation to add
        """
        try:
            self.lock.acquire()
            self.pending.add(reservation=reservation)
        finally:
            self.lock.release()

    def remove_pending(self, *, reservation: ABCReservationMixin):
        """
        Removes a reservation to the pending list.
        @params reservation: reservation to remove
        """
        try:
            self.lock.acquire()
            self.pending.remove(reservation=reservation)
        finally:
            self.lock.release()

    def get_renewing(self, *, cycle: int) -> ReservationSet:
        """
        Returns the reservations that need to be renewed on the specified cycle.
        @params cycle : cycle number
        @returns reservation set with reservations to be renewed on the specified cycle
        """
        try:
            self.lock.acquire()
            return self.renewing.get_all_reservations(cycle=cycle)
        finally:
            self.lock.release()

    def add_renewing(self, *, reservation: ABCReservationMixin, cycle: int):
        """
        Adds a reservation to the renewing list at the given cycle.
        @params reservation : reservation to add
        @params cycle : cycle number
        """
        try:
            self.lock.acquire()
            self.renewing.add_reservation(reservation=reservation, cycle=cycle)
        finally:
            self.lock.release()

    def remove_renewing(self, *, reservation: ABCReservationMixin):
        """
        Removes the reservation from the renewing list.
        @params reservation : reservation to remove
        """
        try:
            self.lock.acquire()
            self.renewing.remove_reservation(reservation=reservation)
        finally:
            self.lock.release()

    def get_holdings(self,
                     *,
                     d: datetime = None,
                     type: ResourceType = None) -> ReservationSet:
        """
        Returns the resources of the specified type held by the client that are active at the specified time instance.
        @params d : datetime instance.
        @params type : resource type
        @returns st of reservations of the specified type that are active at the specified time
        """
        try:
            self.lock.acquire()
            when = None
            if d is not None:
                when = ActorClock.to_milliseconds(when=d)
            return self.holdings.get_reservations(time=when, rtype=type)
        finally:
            self.lock.release()

    def add_holdings(self, *, reservation: ABCReservationMixin,
                     start: datetime, end: datetime):
        """
        Adds a reservation to the holdings list.
        @params reservation : reservation to add
        @params start : start time
        @params end : end time
        """
        try:
            self.lock.acquire()
            self.holdings.add_reservation(
                reservation=reservation,
                start=ActorClock.to_milliseconds(when=start),
                end=ActorClock.to_milliseconds(when=end))
        finally:
            self.lock.release()

    def remove_holdings(self, *, reservation: ABCReservationMixin):
        """
        Removes the reservation from the renewing list.
        @params reservation : reservation to remove
        """
        try:
            self.lock.acquire()
            self.holdings.remove_reservation(reservation=reservation)
        finally:
            self.lock.release()

    def tick(self, *, cycle: int):
        try:
            self.lock.acquire()
            super().tick(cycle=cycle)
            self.renewing.tick(cycle=cycle)
            ms = self.clock.cycle_end_in_millis(cycle=cycle)
            self.holdings.tick(time=ms)
        finally:
            self.lock.release()
Esempio n. 13
0
class ControllerCalendarPolicy(Policy, ABCControllerPolicy):
    """
    The base class for all calendar-based service manager policy implementations.
    """
    def __init__(self):
        super().__init__()
        # calendar
        self.calendar = None
        # Contains reservations for which we may have completed performing
        # bookkeeping actions but may need to wait for some other event to take
        # place before we raise the corresponding event.
        self.pending_notify = ReservationSet()
        # If the actor is initialized
        self.initialized = False
        # If true, the orchestrator will close reservations lazily: it will not
        # issue a close and will wait until the site terminates the lease. The
        # major drawback is that leave actions will not be able to connect to the
        # resources, since the resources will not exist at this time.
        self.lazy_close = False

    def check_pending(self):
        """
        Checks pending operations, and installs successfully completed
        requests in the holdings calendar. Note that the policy module must add
        bids to the pending set, or they may not install in the calendar.
       
        @raises Exception in case of error
        """
        rvset = self.calendar.get_pending()
        if rvset is None:
            return

        for reservation in rvset.values():
            if reservation.is_failed():
                # This reservation has failed. Remove it from the list. This is
                # a separate case, because we may fail but not satisfy the
                # condition of the else statement.
                self.logger.debug(
                    "Removing failed reservation from the pending list: {}".
                    format(reservation))
                self.calendar.remove_pending(reservation=reservation)
                self.pending_notify.remove(reservation=reservation)
            elif reservation.is_no_pending(
            ) and not reservation.is_pending_recover():
                # No pending operation and we are not about the reissue a recovery operation on this reservation.
                self.logger.debug(
                    "Controller pending request completed {}".format(
                        reservation))
                if reservation.is_closed():
                    # do nothing handled by close(IReservation)
                    self.logger.debug("No op")
                elif reservation.is_active_ticketed():
                    # An active reservation extended its ticket.
                    # cancel the current close
                    self.calendar.remove_closing(reservation=reservation)
                    # schedule a new close
                    self.calendar.add_closing(reservation=reservation,
                                              cycle=self.get_close(
                                                  reservation=reservation,
                                                  term=reservation.get_term()))
                    # Add from start to end instead of close. It is possible
                    # that holdings may not accurately reflect the actual
                    # number of resources towards the end of a lease. This is
                    # because we assume that we still have resources even after
                    # an advanceClose. When looking at this value, see if the
                    # reservation has closed.
                    self.calendar.add_holdings(
                        reservation=reservation,
                        start=reservation.get_term().get_new_start_time(),
                        end=reservation.get_term().get_end_time())
                    self.calendar.add_redeeming(
                        reservation=reservation,
                        cycle=self.get_redeem(reservation=reservation))
                    if reservation.is_renewable():
                        cycle = self.get_renew(reservation=reservation)
                        reservation.set_renew_time(time=cycle)
                        reservation.set_dirty()
                        self.calendar.add_renewing(reservation=reservation,
                                                   cycle=cycle)
                    self.pending_notify.remove(reservation=reservation)
                elif reservation.is_ticketed():
                    # The reservation obtained a ticket for the first time
                    self.calendar.add_holdings(
                        reservation=reservation,
                        start=reservation.get_term().get_new_start_time(),
                        end=reservation.get_term().get_end_time())
                    self.calendar.add_redeeming(
                        reservation=reservation,
                        cycle=self.get_redeem(reservation=reservation))
                    self.calendar.add_closing(reservation=reservation,
                                              cycle=self.get_close(
                                                  reservation=reservation,
                                                  term=reservation.get_term()))
                    if reservation.is_renewable():
                        cycle = self.get_renew(reservation=reservation)
                        reservation.set_renew_time(time=cycle)
                        reservation.set_dirty()
                        self.calendar.add_renewing(reservation=reservation,
                                                   cycle=cycle)
                    self.pending_notify.remove(reservation=reservation)
                elif reservation.get_state() == ReservationStates.Active:
                    if self.pending_notify.contains(reservation=reservation):
                        # We are waiting for transfer in operations to complete
                        # so that we can raise the lease complete event.
                        if reservation.is_active_joined():
                            self.pending_notify.remove(reservation=reservation)
                    else:
                        # Just completed a lease call (redeem or extendLease).
                        # We need to remove this reservation from closing,
                        # because we added it using r.getTerm(), and add this
                        # reservation to closing using r.getLeasedTerm() [the
                        # site could have changed the term of the reservation].
                        # This assumes that r.getTerm has not changed in the
                        # mean time. This is true now, since the state machine
                        # does not allow more than one pending operation.
                        # Should we change this, we will need to update the
                        # code below.
                        self.calendar.remove_closing(reservation=reservation)
                        self.calendar.add_closing(
                            reservation=reservation,
                            cycle=self.get_close(
                                reservation=reservation,
                                term=reservation.get_lease_term()))
                        if reservation.get_renew_time() == 0:
                            reservation.set_renew_time(
                                time=(self.actor.get_current_cycle() + 1))
                            reservation.set_dirty()
                            self.calendar.add_renewing(
                                reservation=reservation,
                                cycle=reservation.get_renew_time())

                        if not reservation.is_active_joined():
                            # add to the pending notify list so that we can raise the event
                            # when transfer in operations complete.
                            self.pending_notify.add(reservation=reservation)
                elif reservation.get_state() == ReservationStates.CloseWait or \
                        reservation.get_state() == ReservationStates.Failed:
                    self.pending_notify.remove(reservation=reservation)
                else:
                    self.logger.warning(
                        "Invalid state on reservation. We may be still recovering: {}"
                        .format(reservation))
                    continue

                if not self.pending_notify.contains(reservation=reservation):
                    self.logger.debug(
                        "Removing from pending: {}".format(reservation))
                    self.calendar.remove_pending(reservation=reservation)

    def close(self, *, reservation: ABCReservationMixin):
        # ignore any scheduled/in progress operations
        self.calendar.remove_scheduled_or_in_progress(reservation=reservation)

    def closed(self, *, reservation: ABCReservationMixin):
        # remove the reservation from all calendar structures
        self.calendar.remove_holdings(reservation=reservation)
        self.calendar.remove_redeeming(reservation=reservation)
        self.calendar.remove_renewing(reservation=reservation)
        self.calendar.remove_closing(reservation=reservation)
        self.pending_notify.remove(reservation=reservation)

    def demand(self, *, reservation: ABCClientReservation):
        if not reservation.is_nascent():
            self.logger.error("demand reservation is not fresh")
        else:
            self.calendar.add_demand(reservation=reservation)

    def extend(self, *, reservation: ABCReservationMixin,
               resources: ResourceSet, term: Term):
        # cancel any previously scheduled extends
        self.calendar.remove_renewing(reservation=reservation)
        # do not cancel the close: the extend may fail cancel any pending redeem: we will redeem after the extension
        self.calendar.remove_redeeming(reservation=reservation)
        # There should be no pending operations for this reservation at this time
        # Add to the pending list so that we can track the progress of the reservation
        self.calendar.add_pending(reservation=reservation)

    def finish(self, *, cycle: int):
        super().finish(cycle=cycle)
        self.calendar.tick(cycle=cycle)

    @abstractmethod
    def get_close(self, *, reservation: ABCClientReservation,
                  term: Term) -> int:
        """
        Returns the time that a reservation should be closed.

        @params reservation reservation
        @params term term

        @returns the close time of the reservation (cycle)

        @raises Exception in case of error
        """

    def get_closing(self, *, cycle: int) -> ReservationSet:
        closing = self.calendar.get_closing(cycle=cycle)
        result = ReservationSet()
        for reservation in closing.values():
            if not reservation.is_failed():
                self.calendar.add_pending(reservation=reservation)
                result.add(reservation=reservation)
            else:
                self.logger.warning(
                    "Removing failed reservation from the closing list: {}".
                    format(reservation))
        return result

    @abstractmethod
    def get_redeem(self, *, reservation: ABCClientReservation) -> int:
        """
        Returns the time when the reservation should be redeemed.

        @params reservation the reservation

        @returns the redeem time of the reservation (cycle)

        @raises Exception in case of error
        """

    def get_redeeming(self, *, cycle: int) -> ReservationSet:
        redeeming = self.calendar.get_redeeming(cycle=cycle)
        for reservation in redeeming.values():
            self.calendar.add_pending(reservation=reservation)
        return redeeming

    @abstractmethod
    def get_renew(self, *, reservation: ABCClientReservation) -> int:
        """
        Returns the time when the reservation should be renewed.

        @params reservation the reservation

        @returns the renew time of the reservation (cycle)

        @raises Exception in case of error
        """

    def initialize(self):
        if not self.initialized:
            super().initialize()
            self.calendar = ControllerCalendar(clock=self.clock)
            self.initialized = True

    def is_expired(self, *, reservation: ABCReservationMixin):
        """
        Checks if the reservation has expired.

        @params reservation reservation to check

        @returns true or false
        """
        term = reservation.get_term()
        end = self.clock.cycle(when=term.get_end_time())
        return self.actor.get_current_cycle() > end

    def remove(self, *, reservation: ABCReservationMixin):
        # remove the reservation from the calendar
        self.calendar.remove(reservation=reservation)

    def revisit(self, *, reservation: ABCReservationMixin):
        super().revisit(reservation=reservation)

        if reservation.get_state() == ReservationStates.Nascent:
            self.calendar.add_pending(reservation=reservation)

        elif reservation.get_state() == ReservationStates.Ticketed:

            if reservation.get_pending_state(
            ) == ReservationPendingStates.None_:

                if reservation.is_pending_recover():

                    self.calendar.add_pending(reservation=reservation)

                else:

                    self.calendar.add_redeeming(
                        reservation=reservation,
                        cycle=self.get_redeem(reservation=reservation))

                self.calendar.add_holdings(
                    reservation=reservation,
                    start=reservation.get_term().get_new_start_time(),
                    end=reservation.get_term().get_end_time())

                self.calendar.add_closing(reservation=reservation,
                                          cycle=self.get_close(
                                              reservation=reservation,
                                              term=reservation.get_term()))

                if reservation.is_renewable(
                ) and reservation.get_renew_time() != 0:
                    # Scheduling renewal is a bit tricky, since it may
                    # involve communication with the upstream broker.
                    # However, in some recovery cases, typical in one
                    # container deployment, the broker and the service
                    # manager will be recovering at the same time. In
                    # this case the query may fail and we will have to
                    # fail the reservation.
                    # Our approach here is as follows: we cache the
                    # renew time in the reservation class and persist
                    # it in the database. When we recover, we will
                    # check the renewTime field of the reservation if
                    # it is non-zero, we will use it, otherwise we will
                    # schedule the renew after we get the lease back
                    # from the authority.
                    self.calendar.add_renewing(
                        reservation=reservation,
                        cycle=reservation.get_renew_time())

            elif reservation.get_pending_state(
            ) == ReservationPendingStates.Redeeming:
                raise ControllerException(Constants.INVALID_RECOVERY_STATE)

        elif reservation.get_state() == ReservationStates.Active:
            if reservation.get_pending_state(
            ) == ReservationPendingStates.None_:
                # pending list
                if reservation.is_pending_recover():
                    self.calendar.add_pending(reservation=reservation)
                # renewing
                if reservation.is_renewable():
                    self.calendar.add_renewing(
                        reservation=reservation,
                        cycle=reservation.get_renew_time())
                # holdings
                self.calendar.add_holdings(
                    reservation=reservation,
                    start=reservation.get_term().get_new_start_time(),
                    end=reservation.get_term().get_end_time())
                # closing
                self.calendar.add_closing(
                    reservation=reservation,
                    cycle=self.get_close(reservation=reservation,
                                         term=reservation.get_lease_term()))

            elif reservation.get_pending_state(
            ) == ReservationPendingStates.ExtendingTicket:
                raise ControllerException(Constants.INVALID_RECOVERY_STATE)

        elif reservation.get_state() == ReservationStates.ActiveTicketed:

            if reservation.get_pending_state(
            ) == ReservationPendingStates.None_:
                if reservation.is_pending_recover():
                    self.calendar.add_pending(reservation=reservation)
                else:
                    self.calendar.add_redeeming(
                        reservation=reservation,
                        cycle=self.get_redeem(reservation=reservation))

                # holdings
                self.calendar.add_holdings(
                    reservation=reservation,
                    start=reservation.get_term().get_new_start_time(),
                    end=reservation.get_term().get_end_time())

                # closing
                self.calendar.add_closing(
                    reservation=reservation,
                    cycle=self.get_close(reservation=reservation,
                                         term=reservation.get_lease_term()))

                # renewing
                if reservation.is_renewable():
                    self.calendar.add_renewing(
                        reservation=reservation,
                        cycle=reservation.get_renew_time())

            elif reservation.get_pending_state(
            ) == ReservationPendingStates.ExtendingLease:
                raise ControllerException(Constants.INVALID_RECOVERY_STATE)

    def ticket_satisfies(self, *, requested_resources: ResourceSet,
                         actual_resources: ResourceSet, requested_term: Term,
                         actual_term: Term):
        return

    def update_ticket_complete(self, *, reservation: ABCClientReservation):
        return

    def update_delegation_complete(self,
                                   *,
                                   delegation: ABCDelegation,
                                   reclaim: bool = False):
        return

    def lease_satisfies(self, *, request_resources: ResourceSet,
                        actual_resources: ResourceSet, requested_term: Term,
                        actual_term: Term):
        return
Esempio n. 14
0
class Slice(ABCKernelSlice):
    """
    Slice implementation. A slice has a globally unique identifier, name,
    description, property list, an owning identity, an access control list, and a
    set of reservations.
    This class is used within the Service Manager, which may hold reservations on
    many sites; on the Broker, which may have provided tickets to the slice for
    reservations at many sites; and on the site Authority, where each slice may
    hold multiple reservations for resources at that site.
    """
    def __init__(self, *, slice_id: ID = None, name: str = "unspecified"):
        # Globally unique identifier.
        self.guid = slice_id
        # Slice name. Not required to be globally or locally unique.
        self.name = name
        # Description string. Has only local meaning.
        self.description = "no description"
        # The slice type: inventory or client.
        self.type = SliceTypes.ClientSlice
        # The owner of the slice.
        self.owner = None
        # Resource type associated with this slice. Used when the slice is used to
        # represent an inventory pool.
        self.resource_type = None
        # The reservations in this slice.
        self.reservations = ReservationSet()
        self.delegations = {}
        # Neo4jGraph Id
        self.graph_id = None
        self.graph = None
        self.state_machine = SliceStateMachine(slice_id=slice_id)
        self.dirty = False
        self.config_properties = None
        self.lock = threading.Lock()
        self.lease_end = None
        self.lease_start = None

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['reservations']
        del state['delegations']
        del state['graph']
        del state['lock']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.reservations = ReservationSet()
        self.graph = None
        self.delegations = {}
        self.lock = threading.Lock()

    def set_graph_id(self, graph_id: str):
        self.graph_id = graph_id

    def get_graph_id(self) -> str:
        return self.graph_id

    def set_graph(self, *, graph: ABCPropertyGraph):
        self.graph = graph
        self.set_graph_id(graph_id=self.graph.get_graph_id())

    def get_graph(self) -> ABCPropertyGraph:
        return self.graph

    def clone_request(self) -> ABCSlice:
        result = Slice()
        result.name = self.name
        result.guid = self.guid
        return result

    def get_lease_end(self) -> datetime:
        return self.lease_end

    def get_lease_start(self) -> datetime:
        return self.lease_start

    def get_description(self) -> str:
        return self.description

    def get_name(self) -> str:
        return self.name

    def get_owner(self) -> AuthToken:
        return self.owner

    def get_reservations(self) -> ReservationSet:
        return self.reservations

    def get_delegations(self) -> Dict[str, ABCDelegation]:
        return self.delegations

    def get_reservations_list(self) -> list:
        return self.reservations.values()

    def get_resource_type(self) -> ResourceType:
        return self.resource_type

    def get_slice_id(self) -> ID:
        return self.guid

    def is_broker_client(self) -> bool:
        return self.type == SliceTypes.BrokerClientSlice

    def is_client(self) -> bool:
        return not self.is_inventory()

    def is_inventory(self) -> bool:
        return self.type == SliceTypes.InventorySlice

    def is_empty(self) -> bool:
        return self.reservations.is_empty()

    def prepare(self):
        self.reservations.clear()
        self.delegations.clear()
        self.state_machine.clear()
        self.transition_slice(operation=SliceStateMachine.CREATE)

    def register(self, *, reservation: ABCKernelReservation):
        if self.reservations.contains(rid=reservation.get_reservation_id()):
            raise SliceException(
                "Reservation #{} already exists in slice".format(
                    reservation.get_reservation_id()))

        self.reservations.add(reservation=reservation)

    def register_delegation(self, *, delegation: ABCDelegation):
        if delegation.get_delegation_id() in self.delegations:
            raise SliceException(
                "Delegation #{} already exists in slice".format(
                    delegation.get_delegation_id()))

        self.delegations[delegation.get_delegation_id()] = delegation

    def set_lease_end(self, *, lease_end: datetime):
        self.lease_end = lease_end

    def set_lease_start(self, *, lease_start: datetime):
        self.lease_start = lease_start

    def set_broker_client(self):
        self.type = SliceTypes.BrokerClientSlice

    def set_client(self):
        self.type = SliceTypes.ClientSlice

    def set_description(self, *, description: str):
        self.description = description

    def set_inventory(self, *, value: bool):
        if value:
            self.type = SliceTypes.InventorySlice
        else:
            self.type = SliceTypes.ClientSlice

    def get_slice_type(self) -> SliceTypes:
        return self.type

    def set_name(self, *, name: str):
        self.name = name

    def set_owner(self, *, owner: AuthToken):
        self.owner = owner

    def set_resource_type(self, *, resource_type: ResourceType):
        self.resource_type = resource_type

    def soft_lookup(self, *, rid: ID) -> ABCKernelReservation:
        return self.reservations.get(rid=rid)

    def soft_lookup_delegation(self, *, did: str) -> ABCDelegation:
        return self.delegations.get(did, None)

    def __str__(self):
        msg = "{}({})".format(self.name, str(self.guid))
        if self.graph_id is not None:
            msg += " Graph Id:{}".format(self.graph_id)
        if self.owner is not None:
            if self.owner.get_email() is not None:
                msg += " Owner:{}".format(self.owner.get_email())
            else:
                msg += " Owner:{}".format(self.owner)
        '''
        if self.state_machine is not None:
            msg += " State:{}".format(self.state_machine.get_state())
        '''
        return msg

    def unregister(self, *, reservation: ABCKernelReservation):
        self.reservations.remove(reservation=reservation)

    def unregister_delegation(self, *, delegation: ABCDelegation):
        if delegation.get_delegation_id() in self.delegations:
            self.delegations.pop(delegation.get_delegation_id())

    def get_state(self) -> SliceState:
        return self.state_machine.get_state()

    def set_dirty(self):
        self.dirty = True

    def clear_dirty(self):
        self.dirty = False

    def is_dirty(self) -> bool:
        return self.dirty

    def transition_slice(self, *,
                         operation: SliceOperation) -> Tuple[bool, SliceState]:
        return self.state_machine.transition_slice(
            operation=operation, reservations=self.reservations)

    def is_stable_ok(self) -> bool:
        state_changed, slice_state = self.transition_slice(
            operation=SliceStateMachine.REEVALUATE)

        if slice_state == SliceState.StableOk:
            return True

        return False

    def is_stable_error(self) -> bool:
        state_changed, slice_state = self.transition_slice(
            operation=SliceStateMachine.REEVALUATE)

        if slice_state == SliceState.StableError:
            return True

        return False

    def is_stable(self) -> bool:
        state_changed, slice_state = self.transition_slice(
            operation=SliceStateMachine.REEVALUATE)

        if slice_state == SliceState.StableError or slice_state == SliceState.StableOk:
            return True

        return False

    def is_dead_or_closing(self) -> bool:
        state_changed, slice_state = self.transition_slice(
            operation=SliceStateMachine.REEVALUATE)

        if slice_state == SliceState.Dead or slice_state == SliceState.Closing:
            return True

        return False

    def is_dead(self) -> bool:
        state_changed, slice_state = self.transition_slice(
            operation=SliceStateMachine.REEVALUATE)

        if slice_state == SliceState.Dead:
            return True

        return False

    def set_config_properties(self, *, value: dict):
        self.config_properties = value

    def get_config_properties(self) -> dict:
        return self.config_properties

    def update_slice_graph(self, sliver: BaseSliver, rid: str,
                           reservation_state: str) -> BaseSliver:
        try:
            self.lock.acquire()
            # Update for Orchestrator for Active / Ticketed Reservations
            if sliver is not None and self.graph_id is not None:
                if sliver.reservation_info is None:
                    sliver.reservation_info = ReservationInfo()
                sliver.reservation_info.reservation_id = rid
                sliver.reservation_info.reservation_state = reservation_state
                FimHelper.update_node(graph_id=self.graph_id, sliver=sliver)
            return sliver
        finally:
            self.lock.release()
Esempio n. 15
0
class Controller(ActorMixin, ABCController):
    """
    Implements Controller
    """
    saved_extended_renewable = ReservationSet()

    def __init__(self,
                 *,
                 identity: AuthToken = None,
                 clock: ActorClock = None):
        super().__init__(auth=identity, clock=clock)
        # Recovered reservations that need to obtain tickets.
        self.ticketing = ReservationSet()
        # Recovered reservations that need to extend tickets.
        self.extending_ticket = ReservationSet()
        # Recovered reservations that need to be redeemed.
        self.redeeming = ReservationSet()
        # Recovered reservations that need to extend leases.
        self.extending_lease = ReservationSet()
        # Recovered reservations that need to modify leases
        self.modifying_lease = ReservationSet()
        # Peer registry.
        self.registry = PeerRegistry()
        # initialization status
        self.initialized = False
        self.type = ActorType.Orchestrator

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['recovered']
        del state['wrapper']
        del state['logger']
        del state['clock']
        del state['current_cycle']
        del state['first_tick']
        del state['stopped']
        del state['initialized']
        del state['thread_lock']
        del state['thread']
        del state['timer_queue']
        del state['event_queue']
        del state['reservation_tracker']
        del state['subscription_id']
        del state['actor_main_lock']
        del state['closing']
        del state['message_service']

        del state['ticketing']
        del state['extending_ticket']
        del state['redeeming']
        del state['extending_lease']
        del state['modifying_lease']
        del state['registry']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.recovered = False
        self.wrapper = None
        self.logger = None
        self.clock = None
        self.current_cycle = -1
        self.first_tick = True
        self.stopped = False
        self.initialized = False
        self.thread = None
        self.thread_lock = threading.Lock()
        self.timer_queue = queue.Queue()
        self.event_queue = queue.Queue()
        self.reservation_tracker = None
        self.subscription_id = None
        self.actor_main_lock = threading.Condition()
        self.closing = ReservationSet()
        self.message_service = None

        self.ticketing = ReservationSet()
        self.extending_ticket = ReservationSet()
        self.redeeming = ReservationSet()
        self.extending_lease = ReservationSet()
        self.modifying_lease = ReservationSet()
        self.registry = PeerRegistry()

    def actor_added(self):
        super().actor_added()
        self.registry.actor_added()

    def add_broker(self, *, broker: ABCBrokerProxy):
        self.registry.add_broker(broker=broker)

    def bid(self):
        """
        Bids for resources as dictated by the plugin bidding policy for the
        current cycle.

        @throws Exception in case of error
        """
        # Invoke policy module to select candidates for ticket and extend. Note
        # that candidates structure is discarded when we're done.
        candidates = self.policy.formulate_bids(cycle=self.current_cycle)
        if candidates is not None:
            # Issue new ticket requests.
            ticketing = candidates.get_ticketing()
            if ticketing is not None:
                for ticket in ticketing.values():
                    try:
                        self.wrapper.ticket(reservation=ticket,
                                            destination=self)
                    except Exception as e:
                        self.logger.error(
                            "unexpected ticket failure for #{} {}".format(
                                ticket.get_reservation_id(), e))
                        ticket.fail(
                            message="unexpected ticket failure {}".format(e))

            extending = candidates.get_extending()
            if extending is not None:
                for extend in extending.values():
                    try:
                        self.wrapper.extend_ticket(reservation=extend)
                    except Exception as e:
                        self.logger.error(
                            "unexpected extend failure for #{} {}".format(
                                extend.get_reservation_id(), e))
                        extend.fail(
                            message="unexpected extend failure {}".format(e))

    def claim_delegation_client(self,
                                *,
                                delegation_id: str = None,
                                slice_object: ABCSlice = None,
                                broker: ABCBrokerProxy = None,
                                id_token: str = None) -> ABCDelegation:
        raise ControllerException("Not implemented")

    def reclaim_delegation_client(self,
                                  *,
                                  delegation_id: str = None,
                                  slice_object: ABCSlice = None,
                                  broker: ABCBrokerProxy = None,
                                  id_token: str = None) -> ABCDelegation:
        raise ControllerException("Not implemented")

    def close_expiring(self):
        """
        Issues close requests on all reservations scheduled for closing on the
        current cycle
        """
        rset = self.policy.get_closing(cycle=self.current_cycle)

        if rset is not None and rset.size() > 0:
            self.logger.info(
                "SlottedSM close expiring for cycle {} expiring {}".format(
                    self.current_cycle, rset))
            self.close_reservations(reservations=rset)

    def demand(self, *, rid: ID):
        if rid is None:
            raise ControllerException("Invalid argument")

        reservation = self.get_reservation(rid=rid)

        if reservation is None:
            raise ControllerException("Unknown reservation {}".format(rid))

        self.policy.demand(reservation=reservation)
        reservation.set_policy(policy=self.policy)

    def extend_lease_reservation(self, *,
                                 reservation: ABCControllerReservation):
        """
        Extend Lease for a reservation
        @param reservation reservation
        """
        if not self.recovered:
            self.extending_lease.add(reservation=reservation)
        else:
            self.wrapper.extend_lease(reservation=reservation)

    def extend_lease(self,
                     *,
                     reservation: ABCControllerReservation = None,
                     rset: ReservationSet = None):
        if reservation is not None and rset is not None:
            raise ControllerException(
                "Invalid Arguments: reservation and rset can not be both not None"
            )
        if reservation is None and rset is None:
            raise ControllerException(
                "Invalid Arguments: reservation and rset can not be both None")

        if reservation is not None:
            self.extend_lease_reservation(reservation=reservation)

        if rset is not None:
            for r in rset.values():
                try:
                    if isinstance(r, ABCControllerReservation):
                        self.extend_lease_reservation(reservation=r)
                    else:
                        self.logger.warning(
                            "Reservation #{} cannot extendLease".format(
                                r.get_reservation_id()))
                except Exception as e:
                    self.logger.error(
                        "Could not extend_lease for #{} e={}".format(
                            r.get_reservation_id(), e))

    def extend_ticket_client(self, *, reservation: ABCClientReservation):
        if not self.recovered:
            self.extending_ticket.add(reservation=reservation)
        else:
            self.wrapper.extend_ticket(reservation=reservation)

    def extend_tickets_client(self, *, rset: ReservationSet):
        for reservation in rset.values():
            try:
                if isinstance(reservation, ABCClientReservation):
                    self.extend_ticket_client(reservation=reservation)
                else:
                    self.logger.warning(
                        "Reservation # {} cannot be ticketed".format(
                            reservation.get_reservation_id()))
            except Exception as e:
                self.logger.error("Could not ticket for #{} e: {}".format(
                    reservation.get_reservation_id(), e))

    def get_broker(self, *, guid: ID) -> ABCBrokerProxy:
        return self.registry.get_broker(guid=guid)

    def get_brokers(self) -> list:
        return self.registry.get_brokers()

    def get_default_broker(self) -> ABCBrokerProxy:
        return self.registry.get_default_broker()

    def initialize(self):
        if not self.initialized:
            super().initialize()

            self.registry.set_slices_plugin(plugin=self.plugin)
            self.registry.initialize()

            self.initialized = True

    def process_redeeming(self):
        """
        Issue redeem requests on all reservations scheduled for redeeming on the current cycle
        """
        rset = self.policy.get_redeeming(cycle=self.current_cycle)

        if rset is not None and rset.size() > 0:
            self.logger.info(
                "SlottedController redeem for cycle {} redeeming {}".format(
                    self.current_cycle, rset))

            self.redeem_reservations(rset=rset)

    def redeem(self, *, reservation: ABCControllerReservation):
        if not self.recovered:
            self.redeeming.add(reservation=reservation)
        else:
            self.wrapper.redeem(reservation=reservation)

    def redeem_reservations(self, *, rset: ReservationSet):
        for reservation in rset.values():
            try:
                if isinstance(reservation, ABCControllerReservation):
                    self.redeem(reservation=reservation)
                else:
                    self.logger.warning(
                        "Reservation #{} cannot be redeemed".format(
                            reservation.get_reservation_id()))
            except Exception as e:
                self.logger.error(traceback.format_exc())
                self.logger.error("Could not redeem for #{} {}".format(
                    reservation.get_reservation_id(), e))

    def ticket_client(self, *, reservation: ABCClientReservation):
        if not self.recovered:
            self.ticketing.add(reservation=reservation)
        else:
            self.wrapper.ticket(reservation=reservation, destination=self)

    def tickets_client(self, *, rset: ReservationSet):
        for reservation in rset.values():
            try:
                if isinstance(reservation, ABCClientReservation):
                    self.ticket_client(reservation=reservation)
                else:
                    self.logger.warning(
                        "Reservation #{} cannot be ticketed".format(
                            reservation.get_reservation_id()))
            except Exception as e:
                self.logger.error("Could not ticket for #{} e: {}".format(
                    reservation.get_reservation_id(), e))

    def tick_handler(self):
        self.close_expiring()
        self.process_redeeming()
        self.bid()

    def update_lease(self, *, reservation: ABCReservationMixin, update_data,
                     caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise ControllerException("This actor cannot receive calls")

        self.wrapper.update_lease(reservation=reservation,
                                  update_data=update_data,
                                  caller=caller)

    def update_ticket(self, *, reservation: ABCReservationMixin, update_data,
                      caller: AuthToken):
        if not self.is_recovered() or self.is_stopped():
            raise ControllerException("This actor cannot receive calls")

        self.wrapper.update_ticket(reservation=reservation,
                                   update_data=update_data,
                                   caller=caller)

    def update_delegation(self, *, delegation: ABCDelegation, update_data,
                          caller: AuthToken):
        raise ControllerException("Not supported in controller")

    def modify(self, *, reservation_id: ID, modify_properties: dict):
        if reservation_id is None or modify_properties is None:
            self.logger.error(
                "modifyProperties argument is null or non-existing reservation"
            )

        rc = None
        try:
            rc = self.get_reservation(rid=reservation_id)
        except Exception as e:
            self.logger.error("Could not find reservation #{} e: {}".format(
                reservation_id, e))

        if rc is None:
            raise ControllerException(
                "Unknown reservation: {}".format(reservation_id))

        if rc.get_resources() is not None:
            # TODO
            print("TODO")
        else:
            self.logger.warning(
                "There are no approved resources for {}, no modify properties will be added"
                .format(reservation_id))

        if not self.recovered:
            self.modifying_lease.add(reservation=rc)
        else:
            self.wrapper.modify_lease(reservation=rc)

    def save_extending_renewable(self):
        """
        For recovery, mark extending reservations renewable or the opposite
        and save this, then restore afterwards
        """
        for reservation in self.extending_ticket.values():
            try:
                if isinstance(reservation, ABCClientReservation):
                    if not reservation.get_renewable():
                        reservation.set_renewable(renewable=True)
                        self.saved_extended_renewable.add(
                            reservation=reservation)
                else:
                    self.logger.warning(
                        "Reservation #{} cannot be remarked".format(
                            reservation.get_reservation_id()))
            except Exception as e:
                self.logger.error(
                    "Could not mark ticket renewable for #{} e: {}".format(
                        reservation.get_reservation_id(), e))

    def restore_extending_renewable(self):
        """
        Restore the value of renewable field after recovery if we changed it
        """
        for reservation in self.saved_extended_renewable.values():
            try:
                reservation.set_renewable(renewable=False)
            except Exception as e:
                self.logger.error(
                    "Could not remark ticket non renewable for #{} e: {}".
                    format(reservation.get_reservation_id(), e))

    def issue_delayed(self):
        super().issue_delayed()
        self.tickets_client(rset=self.ticketing)
        self.ticketing.clear()

        self.save_extending_renewable()
        self.extend_tickets_client(rset=self.extending_ticket)
        self.extending_ticket.clear()

        self.redeem_reservations(rset=self.redeeming)
        self.redeeming.clear()

        self.extend_lease(rset=self.extending_lease)
        self.extending_lease.clear()

    @staticmethod
    def get_management_object_class() -> str:
        return ControllerManagementObject.__name__

    @staticmethod
    def get_management_object_module() -> str:
        return ControllerManagementObject.__module__

    @staticmethod
    def get_kafka_service_class() -> str:
        return ControllerService.__name__

    @staticmethod
    def get_kafka_service_module() -> str:
        return ControllerService.__module__

    @staticmethod
    def get_mgmt_kafka_service_class() -> str:
        return KafkaControllerService.__name__

    @staticmethod
    def get_mgmt_kafka_service_module() -> str:
        return KafkaControllerService.__module__
Esempio n. 16
0
class ActorMixin(ABCActorMixin):
    """
    Actor is the base class for all actor implementations
    """
    DefaultDescription = "no description"

    actor_count = 0

    def __init__(self, *, auth: AuthToken = None, clock: ActorClock = None):
        # Globally unique identifier for this actor.
        self.guid = ID()
        # Actor name.
        self.name = None
        # Actor type code.
        self.type = ActorType.All
        # Actor description.
        self.description = self.DefaultDescription
        # Identity object representing this actor.
        self.identity = auth
        # Actor policy object.
        self.policy = None
        # Actor plugin
        self.plugin = None
        # True if this actor has completed the recovery phase.
        self.recovered = False
        # The kernel wrapper.
        self.wrapper = None
        # logger
        self.logger = None
        # Factory for term.
        self.clock = clock
        # current cycle
        self.current_cycle = -1
        # True if the current tick is the first tick this actor has received.
        self.first_tick = True
        # Set to true when the actor is stopped.
        self.stopped = False
        # Initialization status.
        self.initialized = False
        # Contains a reference to the thread currently executing the timer handler.
        # This field is set at the entry to and clear at the exit.
        # The primary use of the field is to handle correctly stopping the actor.
        self.thread = None
        # A queue of timers that have fired and need to be processed.
        self.timer_queue = queue.Queue()
        self.event_queue = queue.Queue()
        self.reservation_tracker = None
        self.subscription_id = None
        # Reservations to close once recovery is complete.
        self.closing = ReservationSet()

        self.thread_lock = threading.Lock()
        self.actor_main_lock = threading.Condition()
        self.message_service = None

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['recovered']
        del state['wrapper']
        del state['logger']
        del state['clock']
        del state['current_cycle']
        del state['first_tick']
        del state['stopped']
        del state['initialized']
        del state['thread_lock']
        del state['thread']
        del state['timer_queue']
        del state['event_queue']
        del state['reservation_tracker']
        del state['subscription_id']
        del state['actor_main_lock']
        del state['closing']
        del state['message_service']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.recovered = False
        self.wrapper = None
        self.logger = None
        self.clock = None
        self.current_cycle = -1
        self.first_tick = True
        self.stopped = False
        self.initialized = False
        self.thread = None
        self.thread_lock = threading.Lock()
        self.timer_queue = queue.Queue()
        self.event_queue = queue.Queue()
        self.subscription_id = None
        self.actor_main_lock = threading.Condition()
        self.closing = ReservationSet()
        self.message_service = None
        self.policy.set_actor(actor=self)

    def actor_added(self):
        self.plugin.actor_added()

    def actor_removed(self):
        return

    def fail(self, *, rid: ID, message: str):
        self.wrapper.fail(rid=rid, message=message)

    def fail_delegation(self, *, did: str, message: str):
        self.wrapper.fail_delegation(did=did, message=message)

    def close_by_rid(self, *, rid: ID):
        self.wrapper.close(rid=rid)

    def close(self, *, reservation: ABCReservationMixin):
        if reservation is not None:
            if not self.recovered:
                self.logger.debug(
                    "Adding reservation: {} to closing list".format(
                        reservation.get_reservation_id()))
                self.closing.add(reservation=reservation)
            else:
                self.logger.debug("Closing reservation: {}".format(
                    reservation.get_reservation_id()))
                self.wrapper.close(rid=reservation.get_reservation_id())

    def close_slice_reservations(self, *, slice_id: ID):
        self.wrapper.close_slice_reservations(slice_id=slice_id)

    def close_reservations(self, *, reservations: ReservationSet):
        for reservation in reservations.values():
            try:
                self.logger.debug("Closing reservation: {}".format(
                    reservation.get_reservation_id()))
                self.close(reservation=reservation)
            except Exception as e:
                self.logger.error(traceback.format_exc())
                self.logger.error("Could not close for #{} {}".format(
                    reservation.get_reservation_id(), e))

    def error(self, *, err: str):
        """
        Logs and propagates a general error.

        @param err
                   log/exception message.
        @throws Exception
                    always
        """
        self.logger.error(err)
        raise ActorException(err)

    def extend(self, *, rid: ID, resources: ResourceSet, term: Term):
        self.wrapper.extend_reservation(rid=rid,
                                        resources=resources,
                                        term=term)

    def external_tick(self, *, cycle: int):
        self.logger.info("External Tick start cycle: {}".format(cycle))

        class TickEvent(ABCActorEvent):
            def __init__(self, *, base, cycle: int):
                self.base = base
                self.cycle = cycle

            def __str__(self):
                return "{} {}".format(self.base, self.cycle)

            def process(self):
                self.base.actor_tick(cycle=self.cycle)

        self.queue_event(incoming=TickEvent(base=self, cycle=cycle))
        self.logger.info("External Tick end cycle: {}".format(cycle))

    def actor_tick(self, *, cycle: int):
        """
        Actor Tick
        :param cycle: cycle
        :return:
        """
        try:
            if not self.recovered:
                self.logger.warning(
                    "Tick for an actor that has not completed recovery")
                return
            current_cycle = 0
            if self.first_tick:
                current_cycle = cycle
            else:
                current_cycle = self.current_cycle + 1

            while current_cycle <= cycle:
                self.logger.info("actor_tick: {} start".format(current_cycle))
                self.current_cycle = current_cycle
                self.policy.prepare(cycle=self.current_cycle)

                if self.first_tick:
                    self.reset()

                self.tick_handler()
                self.policy.finish(cycle=self.current_cycle)

                self.wrapper.tick()

                self.first_tick = False
                self.logger.info("actor_tick: {} end".format(current_cycle))
                current_cycle += 1
        except Exception as e:
            self.logger.debug(traceback.format_exc())
            raise e

    def get_actor_clock(self) -> ActorClock:
        return self.clock

    def get_client_slices(self) -> List[ABCSlice]:
        return self.wrapper.get_client_slices()

    def get_current_cycle(self) -> int:
        return self.current_cycle

    def get_description(self) -> str:
        return self.description

    def get_guid(self) -> ID:
        if self.identity is not None:
            return self.identity.get_guid()
        return None

    def get_identity(self) -> AuthToken:
        return self.identity

    def get_inventory_slices(self) -> List[ABCSlice]:
        """
        Get inventory slices
        @return inventory slices
        """
        return self.wrapper.get_inventory_slices()

    def get_logger(self):
        return self.logger

    def get_name(self) -> str:
        return self.name

    def get_policy(self) -> ABCPolicy:
        return self.policy

    def get_delegation(self, *, did: str) -> ABCDelegation:
        return self.wrapper.get_delegation(did=did)

    def get_reservation(self, *, rid: ID) -> ABCReservationMixin:
        return self.wrapper.get_reservation(rid=rid)

    def get_reservations(self, *, slice_id: ID) -> List[ABCReservationMixin]:
        return self.wrapper.get_reservations(slice_id=slice_id)

    def get_plugin(self):
        return self.plugin

    def get_slice(self, *, slice_id: ID) -> ABCSlice:
        return self.wrapper.get_slice(slice_id=slice_id)

    def get_slices(self):
        return self.wrapper.get_slices()

    def get_type(self) -> ActorType:
        return self.type

    def initialize(self):
        from fabric_cf.actor.core.container.globals import GlobalsSingleton

        if not self.initialized:
            if self.identity is None or self.plugin is None or self.policy is None:
                raise ActorException(
                    f"The actor is not properly created: identity: {self.identity} "
                    f"plugin: {self.plugin} policy: {self.policy}")

            if self.name is None:
                self.name = self.identity.get_name()

            if self.name is None:
                raise ActorException(
                    "The actor is not properly created: no name")

            if self.clock is None:
                self.clock = GlobalsSingleton.get().get_container(
                ).get_actor_clock()

            if self.clock is None:
                raise ActorException(
                    "The actor is not properly created: no clock")

            if self.logger is None:
                self.logger = GlobalsSingleton.get().get_logger()

            self.plugin.set_actor(actor=self)
            self.plugin.set_logger(logger=self.logger)
            self.plugin.initialize()

            self.policy.set_actor(actor=self)
            self.policy.initialize()
            self.policy.set_logger(logger=self.logger)

            self.wrapper = KernelWrapper(actor=self,
                                         plugin=self.plugin,
                                         policy=self.policy)

            self.current_cycle = -1

            self.setup_message_service()

            self.initialized = True

    def is_recovered(self) -> bool:
        return self.recovered

    def is_stopped(self) -> bool:
        return self.stopped

    def query(self,
              *,
              query: dict = None,
              caller: AuthToken = None,
              actor_proxy: ABCActorProxy = None,
              handler: ABCQueryResponseHandler = None,
              id_token: str = None) -> dict:
        """
        Query an actor
        @param query query
        @param caller caller
        @param actor_proxy actor proxy
        @param handler response handler
        @param id_token identity token
        """
        if actor_proxy is None and handler is None:
            return self.wrapper.query(properties=query,
                                      caller=caller,
                                      id_token=id_token)
        else:
            callback = Proxy.get_callback(actor=self,
                                          protocol=actor_proxy.get_type())
            RPCManagerSingleton.get().query(actor=self,
                                            remote_actor=actor_proxy,
                                            callback=callback,
                                            query=query,
                                            handler=handler,
                                            id_token=id_token)
            return None

    def recover(self):
        """
        Recover
        """
        self.logger.info("Starting recovery")
        self.recovery_starting()
        self.logger.debug("Recovering inventory slices")

        inventory_slices = self.plugin.get_database().get_inventory_slices()
        self.logger.debug("Found {} inventory slices".format(
            len(inventory_slices)))
        self.recover_slices(slices=inventory_slices)
        self.logger.debug("Recovery of inventory slices complete")

        self.logger.debug("Recovering client slices")
        client_slices = self.plugin.get_database().get_client_slices()
        self.logger.debug("Found {} client slices".format(len(client_slices)))
        self.recover_slices(slices=client_slices)
        self.logger.debug("Recovery of client slices complete")

        self.recovered = True

        self.recovery_ended()
        self.logger.info("Recovery complete")

    def recovery_starting(self):
        """
        Recovery starting
        """
        self.plugin.recovery_starting()
        self.policy.recovery_starting()

    def recovery_ended(self):
        """
        Recovery ended
        """
        self.plugin.recovery_ended()
        self.policy.recovery_ended()
        from fabric_cf.actor.core.container.globals import GlobalsSingleton
        if GlobalsSingleton.get().can_reload_model():
            GlobalsSingleton.get().delete_reload_model_state_file()

    def recover_slices(self, *, slices: List[ABCSlice]):
        """
        Recover slices
        @param slices slices
        """
        for s in slices:
            try:
                self.recover_slice(slice_obj=s)
            except Exception as e:
                self.logger.error(traceback.format_exc())
                self.logger.error(
                    "Error in recoverSlice for property list {}".format(e))
                if s.is_inventory():
                    raise e

    def recover_broker_slice(self, *, slice_obj: ABCSlice):
        """
        Recover broker slice at the AM, do the following if the model.reload file is detected
        - Close the existing delegations
        - Create the new delegations from the reloaded ARM
        - Add the delegations to the Broker Slice

        @param slice_obj Slice object
        """
        if self.get_type() != ActorType.Authority:
            return False

        if not slice_obj.is_broker_client():
            return False

        from fabric_cf.actor.core.container.globals import GlobalsSingleton
        if not GlobalsSingleton.get().can_reload_model():
            return False

        self.logger.info(
            f"Closing old delegations and adding new delegations to the slice: {slice_obj}!"
        )
        delegation_names = []

        try:
            delegations = self.plugin.get_database(
            ).get_delegations_by_slice_id(slice_id=slice_obj.get_slice_id())
        except Exception as e:
            self.logger.error(e)
            raise ActorException(
                f"Could not fetch delegations records for slice {slice_obj} from database"
            )

        for d in delegations:
            self.logger.info(f"Closing delegation: {d}!")
            d.set_graph(graph=None)
            d.transition(prefix="closed as part of recovers",
                         state=DelegationState.Closed)
            delegation_names.append(d.get_delegation_name())
            self.plugin.get_database().update_delegation(delegation=d)

        adms = self.policy.aggregate_resource_model.generate_adms()

        # Create new delegations and add to the broker slice; they will be re-registered with the policy in the recovery
        for name in delegation_names:
            new_delegation_graph = adms.get(name)
            dlg_obj = DelegationFactory.create(
                did=new_delegation_graph.get_graph_id(),
                slice_id=slice_obj.get_slice_id(),
                delegation_name=name)
            dlg_obj.set_slice_object(slice_object=slice_obj)
            dlg_obj.set_graph(graph=new_delegation_graph)
            dlg_obj.transition(prefix="Reload Model",
                               state=DelegationState.Delegated)
            self.plugin.get_database().add_delegation(delegation=dlg_obj)

    def recover_inventory_slice(self, *, slice_obj: ABCSlice) -> bool:
        """
        Check and Reload ARM for an inventory slice for an AM
        @param slice_obj slice object
        @return True if ARM was reloaded; otherwise False
        """
        if self.get_type() != ActorType.Authority:
            return False

        if not slice_obj.is_inventory():
            return False

        # Check and Reload ARM if needed
        from fabric_cf.actor.core.container.globals import GlobalsSingleton
        arm_graph = GlobalsSingleton.get().check_and_reload_model(
            graph_id=slice_obj.get_graph_id())
        if arm_graph is not None:
            slice_obj.set_graph(graph=arm_graph)

        return arm_graph is not None

    def recover_slice(self, *, slice_obj: ABCSlice):
        """
        Recover slice
        @param slice_obj slice_obj
        """
        slice_id = slice_obj.get_slice_id()

        if self.get_slice(slice_id=slice_id) is not None:
            self.logger.debug("Found slice_id: {} slice:{}".format(
                slice_id, slice_obj))
        else:
            self.logger.info("Recovering slice: {}".format(slice_id))
            self.recover_inventory_slice(slice_obj=slice_obj)

            self.recover_broker_slice(slice_obj=slice_obj)

            self.logger.debug("Informing the plugin about the slice")
            self.plugin.revisit(slice_obj=slice_obj)

            self.logger.debug("Registering slice: {}".format(slice_id))
            self.re_register_slice(slice_object=slice_obj)

            self.logger.debug(
                "Recovering reservations in slice: {}".format(slice_id))
            self.recover_reservations(slice_obj=slice_obj)

            self.logger.debug(
                "Recovering delegations in slice: {}".format(slice_id))
            self.recover_delegations(slice_obj=slice_obj)

            self.logger.info("Recovery of slice {} complete".format(slice_id))

    def recover_reservations(self, *, slice_obj: ABCSlice):
        """
        Recover reservations
        @param slice_obj slice object
        """
        self.logger.info(
            "Starting to recover reservations in slice {}({})".format(
                slice_obj.get_name(), slice_obj.get_slice_id()))
        reservations = None
        try:
            reservations = self.plugin.get_database(
            ).get_reservations_by_slice_id(slice_id=slice_obj.get_slice_id())
        except Exception as e:
            self.logger.error(e)
            raise ActorException(
                "Could not fetch reservation records for slice {}({}) from database"
                .format(slice_obj.get_name(), slice_obj.get_slice_id()))

        self.logger.debug("There are {} reservations(s) in slice".format(
            len(reservations)))

        for r in reservations:
            try:
                self.recover_reservation(r=r, slice_obj=slice_obj)
            except Exception as e:
                self.logger.error(
                    "Unexpected error while recovering reservation {}".format(
                        e))

        self.logger.info(
            "Recovery for reservations in slice {} completed".format(
                slice_obj))

    def recover_reservation(self, *, r: ABCReservationMixin,
                            slice_obj: ABCSlice):
        """
        Recover reservation
        @param r reservation
        @param slice_obj slice object
        """
        try:
            r.restore(actor=self, slice_obj=slice_obj)

            self.logger.info("Found reservation # {} in state {}".format(
                r.get_reservation_id(), r.get_reservation_state()))
            if r.is_closed():
                self.logger.info(
                    "Reservation #{} is closed. Nothing to recover.".format(
                        r.get_reservation_id()))
                return

            self.logger.info("Recovering reservation #{}".format(
                r.get_reservation_id()))
            self.logger.debug("Recovering reservation object r={}".format(r))

            self.logger.debug("Registering the reservation with the actor")
            self.re_register(reservation=r)

            self.logger.info(r)

            self.logger.debug("Revisiting with the Plugin")

            self.plugin.revisit(reservation=r)

            self.logger.info(r)

            self.logger.debug("Revisiting with the actor policy")
            self.policy.revisit(reservation=r)

            self.logger.info("Recovered reservation #{}".format(
                r.get_reservation_id()))
        except Exception as e:
            self.logger.error(traceback.format_exc())
            self.logger.error(
                "Exception occurred in recovering reservation e={}".format(e))
            raise ActorException("Could not recover Reservation #{}".format(r))

    def recover_delegations(self, *, slice_obj: ABCSlice):
        """
        Recover delegations for a slice
        @param slice_obj slice object
        """
        self.logger.info(
            "Starting to recover delegations in slice {}({})".format(
                slice_obj.get_name(), slice_obj.get_slice_id()))
        try:
            delegations = self.plugin.get_database(
            ).get_delegations_by_slice_id(slice_id=slice_obj.get_slice_id())
        except Exception as e:
            self.logger.error(e)
            raise ActorException(
                "Could not fetch delegations records for slice {}({}) from database"
                .format(slice_obj.get_name(), slice_obj.get_slice_id()))

        self.logger.debug("There are {} delegations(s) in slice".format(
            len(delegations)))

        for d in delegations:
            try:
                self.logger.info("Delegation has properties: {}".format(d))
                self.recover_delegation(d=d, slice_obj=slice_obj)
            except Exception as e:
                self.logger.error(
                    "Unexpected error while recovering delegation {}".format(
                        e))

        self.logger.info(
            "Recovery for delegations in slice {} completed".format(slice_obj))

    def recover_delegation(self, *, d: ABCDelegation, slice_obj: ABCSlice):
        """
        Recover delegation
        @param d delegation
        @param slice_obj slice object
        """
        try:

            d.restore(actor=self, slice_obj=slice_obj)

            self.logger.info("Found delegation # {} in state {}".format(
                d.get_delegation_id(), d.get_state_name()))
            if d.is_closed():
                self.logger.info(
                    "Delegation #{} is closed. Nothing to recover.".format(
                        d.get_delegation_id()))
                return

            self.logger.info("Recovering delegation #{}".format(
                d.get_delegation_id()))
            self.logger.debug("Recovering delegation object d={}".format(d))

            self.logger.debug("Registering the delegation with the actor")
            self.re_register_delegation(delegation=d)

            self.logger.info(d)

            self.logger.debug("Revisiting with the Plugin")

            self.plugin.revisit(delegation=d)

            self.logger.info(d)

            self.logger.debug("Revisiting with the actor policy")
            self.policy.revisit_delegation(delegation=d)

            self.logger.info("Recovered delegation #{}".format(
                d.get_delegation_id()))
        except Exception as e:
            self.logger.error(traceback.format_exc())
            self.logger.error(
                "Exception occurred in recovering delegation e={}".format(e))
            raise ActorException("Could not recover delegation #{}".format(d))

    def register(self, *, reservation: ABCReservationMixin):
        self.wrapper.register_reservation(reservation=reservation)

    def register_slice(self, *, slice_object: ABCSlice):
        self.wrapper.register_slice(slice_object=slice_object)

    def register_delegation(self, *, delegation: ABCDelegation):
        self.wrapper.register_delegation(delegation=delegation)

    def remove_reservation(self,
                           *,
                           reservation: ABCReservationMixin = None,
                           rid: ID = None):
        if reservation is not None:
            self.wrapper.remove_reservation(
                rid=reservation.get_reservation_id())

        if rid is not None:
            self.wrapper.remove_reservation(rid=rid)

    def remove_slice(self, *, slice_object: ABCSlice):
        self.wrapper.remove_slice(slice_id=slice_object.get_slice_id())

    def remove_slice_by_slice_id(self, *, slice_id: ID):
        self.wrapper.remove_slice(slice_id=slice_id)

    def re_register_delegation(self, *, delegation: ABCDelegation):
        self.wrapper.re_register_delegation(delegation=delegation)

    def re_register(self, *, reservation: ABCReservationMixin):
        self.wrapper.re_register_reservation(reservation=reservation)

    def re_register_slice(self, *, slice_object: ABCSlice):
        self.wrapper.re_register_slice(slice_object=slice_object)

    def issue_delayed(self):
        """
        Issues delayed operations
        """
        assert self.recovered
        self.close_reservations(reservations=self.closing)
        self.closing.clear()

    def reset(self):
        """
        Reset an actor
        """
        self.issue_delayed()
        self.policy.reset()

    def set_actor_clock(self, *, clock):
        """
        Set actor clock
        @param clock clock
        """
        self.clock = clock

    def set_description(self, *, description: str):
        """
        Set description
        @param description description
        """
        self.description = description

    def set_identity(self, *, token: AuthToken):
        """
        Set identity
        @param token token
        """
        self.identity = token
        self.name = self.identity.get_name()
        self.guid = token.get_guid()

    def set_policy(self, *, policy):
        """
        Set policy
        @param policy policy
        """
        self.policy = policy

    def set_recovered(self, *, value: bool):
        """
        Set recovered flag
        @param value value
        """
        self.recovered = value

    def set_plugin(self, *, plugin):
        """
        Set plugin
        @param plugin
        """
        self.plugin = plugin

    def set_stopped(self, *, value: bool):
        """
        Set stopped flag
        @param value value
        """
        self.stopped = value

    def is_on_actor_thread(self) -> bool:
        """
        Check if running on actor thread
        @return true if running on actor thread, false otherwise
        """
        result = False
        try:
            self.thread_lock.acquire()
            result = self.thread == threading.current_thread()
        finally:
            self.thread_lock.release()
        return result

    def execute_on_actor_thread_and_wait(self, *, runnable: ABCActorRunnable):
        """
        Execute an incoming action on actor thread
        @param runnable incoming action/operation
        """
        if self.is_on_actor_thread():
            return runnable.run()
        else:
            status = ExecutionStatus()
            event = ActorEvent(status=status, runnable=runnable)

            self.queue_event(incoming=event)

            with status.lock:
                while not status.done:
                    status.lock.wait()

            if status.exception is not None:
                raise status.exception

            return status.result

    def run(self):
        """
        Actor run function for actor thread
        """
        try:
            self.logger.info("Actor Main Thread started")
            self.actor_count -= 1
            self.actor_main()
        except Exception as e:
            self.logger.error(f"Unexpected error {e}")
            self.logger.error(traceback.format_exc())
        finally:
            self.logger.info("Actor Main Thread exited")

    def start(self):
        """
        Start an Actor
        """
        try:
            self.thread_lock.acquire()
            if self.thread is not None:
                raise ActorException("This actor has already been started")

            self.thread = threading.Thread(target=self.run)
            self.thread.setName(self.get_name())
            self.thread.setDaemon(True)
            self.thread.start()
        finally:
            self.thread_lock.release()

        self.message_service.start()

        if self.plugin.get_handler_processor() is not None:
            self.plugin.get_handler_processor().start()

    def stop(self):
        """
        Stop an actor
        """
        self.stopped = True
        self.message_service.stop()
        try:
            self.thread_lock.acquire()
            temp = self.thread
            self.thread = None
            if temp is not None:
                self.logger.warning(
                    "It seems that the actor thread is running. Interrupting it"
                )
                try:
                    # TODO find equivalent of interrupt
                    with self.actor_main_lock:
                        self.actor_main_lock.notify_all()
                    temp.join()
                except Exception as e:
                    self.logger.error(
                        "Could not join actor thread {}".format(e))
                    self.logger.error(traceback.format_exc())
                finally:
                    self.thread_lock.release()
        finally:
            if self.thread_lock is not None and self.thread_lock.locked():
                self.thread_lock.release()

        if self.plugin.get_handler_processor() is not None:
            self.plugin.get_handler_processor().shutdown()

    def tick_handler(self):
        """
        Tick handler
        """

    def handle_failed_rpc(self, *, rid: ID, rpc: FailedRPC):
        """
        Handler failed rpc
        """
        self.wrapper.process_failed_rpc(rid=rid, rpc=rpc)

    def __str__(self):
        return "actor: [{}/{}]".format(self.name, self.guid)

    def unregister(self, *, reservation: ABCReservationMixin, rid: ID):
        """
        Unregister reservation
        @param reservation reservation
        @param rid reservation id
        """
        if reservation is not None:
            self.wrapper.unregister_reservation(
                rid=reservation.get_reservation_id())

        if rid is not None:
            self.wrapper.unregister_reservation(rid=rid)

    def unregister_slice(self, *, slice_object: ABCSlice):
        """
        Unregister slice
        @param slice_obj slice object
        """
        self.wrapper.unregister_slice(slice_id=slice_object.get_slice_id())

    def unregister_slice_by_slice_id(self, *, slice_id: ID):
        """
        Unregister slice by slice id
        @param slice_id slice id
        """
        self.wrapper.unregister_slice(slice_id=slice_id)

    def queue_timer(self, timer: ABCTimerTask):
        """
        Queue an event on Actor timer queue
        """
        with self.actor_main_lock:
            self.timer_queue.put_nowait(timer)
            self.logger.debug("Added timer to timer queue {}".format(
                timer.__class__.__name__))
            self.actor_main_lock.notify_all()

    def queue_event(self, *, incoming: ABCActorEvent):
        """
        Queue an even on Actor Event Queue
        """
        with self.actor_main_lock:
            self.event_queue.put_nowait(incoming)
            self.logger.debug("Added event to event queue {}".format(
                incoming.__class__.__name__))
            self.actor_main_lock.notify_all()

    def await_no_pending_reservations(self):
        """
        Await until no pending reservations
        """
        self.wrapper.await_nothing_pending()

    def actor_main(self):
        """
        Actor Main loop
        """
        while True:
            events = []
            timers = []

            with self.actor_main_lock:

                while self.event_queue.empty() and self.timer_queue.empty(
                ) and not self.stopped:
                    try:
                        self.actor_main_lock.wait()
                    except InterruptedError as e:
                        self.logger.info("Actor thread interrupted. Exiting")
                        return

                if self.stopped:
                    self.logger.info("Actor exiting")
                    return

                if not self.event_queue.empty():
                    try:
                        for event in IterableQueue(
                                source_queue=self.event_queue):
                            events.append(event)
                    except Exception as e:
                        self.logger.error(
                            f"Error while adding event to event queue! e: {e}")
                        self.logger.error(traceback.format_exc())

                if not self.timer_queue.empty():
                    try:
                        for timer in IterableQueue(
                                source_queue=self.timer_queue):
                            timers.append(timer)
                    except Exception as e:
                        self.logger.error(
                            f"Error while adding event to event queue! e: {e}")
                        self.logger.error(traceback.format_exc())

                self.actor_main_lock.notify_all()

            if len(events) > 0:
                self.logger.debug(f"Processing {len(events)} events")
                for event in events:
                    #self.logger.debug("Processing event of type {}".format(type(e)))
                    #self.logger.debug("Processing event {}".format(e))
                    try:
                        event.process()
                    except Exception as e:
                        self.logger.error(
                            f"Error while processing event {type(event)}, {e}")
                        self.logger.error(traceback.format_exc())

            if len(timers) > 0:
                self.logger.debug(f"Processing {len(timers)} timers")
                for t in timers:
                    try:
                        t.execute()
                    except Exception as e:
                        self.logger.error(
                            f"Error while processing a timer {type(t)}, {e}")
                        self.logger.error(traceback.format_exc())

    def setup_message_service(self):
        """
        Set up Message Service for incoming Kafka Messages
        """
        try:
            # Kafka Proxy Service object
            module_name = self.get_kafka_service_module()
            class_name = self.get_kafka_service_class()
            kafka_service = ReflectionUtils.create_instance_with_params(
                module_name=module_name, class_name=class_name)(actor=self)

            # Kafka Management Service object
            module_name = self.get_mgmt_kafka_service_module()
            class_name = self.get_mgmt_kafka_service_class()
            kafka_mgmt_service = ReflectionUtils.create_instance_with_params(
                module_name=module_name, class_name=class_name)()
            kafka_mgmt_service.set_logger(logger=self.logger)

            # Incoming Message Service
            from fabric_cf.actor.core.container.globals import GlobalsSingleton
            config = GlobalsSingleton.get().get_config()
            topic = config.get_actor().get_kafka_topic()
            topics = [topic]
            consumer_conf = GlobalsSingleton.get().get_kafka_config_consumer()
            self.message_service = MessageService(
                kafka_service=kafka_service,
                kafka_mgmt_service=kafka_mgmt_service,
                consumer_conf=consumer_conf,
                key_schema_location=GlobalsSingleton.get().get_config(
                ).get_kafka_key_schema_location(),
                value_schema_location=GlobalsSingleton.get().get_config(
                ).get_kafka_value_schema_location(),
                topics=topics,
                logger=self.logger)
        except Exception as e:
            self.logger.error(traceback.format_exc())
            self.logger.error("Failed to setup message service e={}".format(e))
            raise e

    def set_logger(self, logger):
        self.logger = logger
        if self.policy is not None:
            self.policy.set_logger(logger=logger)
        if self.plugin is not None:
            self.plugin.set_logger(logger=logger)

    def load_model(self, *, graph_id: str):
        return