def test_resource_analyzer_edge_case():
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0  # limit reached
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.is_edge_case(resource)
    assert ra.can_request(resource) == (False, ResourceStates.EdgeError)
Example #2
0
def test_resource_analyzer_edge_case():
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0  # limit reached
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds(
    ) - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.is_edge_case(resource)
    assert ra.can_request(resource) == (False, ResourceStates.EdgeError)
Example #3
0
def test_limit_reached_reset_infuture():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.WaitingForReset)
def test_limit_reached_reset_infuture():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.WaitingForReset)
def test_resource_with_owner():
    """
    Scenario: resource has an owner.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders(), owner="TestProcessor")
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.HasOwner)
Example #6
0
 def __init__(self,
              name,
              parent_logger=None,
              enable_service_recovery=False):
     BaseService.__init__(self,
                          name,
                          parent_logger=parent_logger,
                          enable_service_recovery=enable_service_recovery)
     self.analyzer = ResourceAnalyzer("resource-analyzer",
                                      parent_logger=self.log)
     self.timing_sorter = ResourceTimingSorter("timing-sorter",
                                               parent_logger=self.log)
     self.queue = None
Example #7
0
def test_limit_reached_reset_ready():
    """
    Scenario: limit is 0, with time_to_reset passed
    Should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() - 2000  # in the past
    # timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (True, None)
def test_limit_reached_reset_ready():
    """
    Scenario: limit is 0, with time_to_reset passed
    Should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() - 2000  # in the past
    # timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (True, None)
def test_with_limit_remaining():
    """
    Scenario: limit is 1000.
    should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource)
Example #10
0
def test_with_limit_remaining():
    """
    Scenario: limit is 1000.
    should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds(
    ) - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource)
def test_waiting_for_interval():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future
    timings.update_timestamp()
    timings.update_interval_timestamp()
    timings.interval = 10000

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.WaitingForInterval)
Example #12
0
def test_resource_with_owner():
    """
    Scenario: resource has an owner.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds(
    ) - 1000  # was done after reset

    resource = Resource("mock://test",
                        timings,
                        ResourceHeaders(),
                        owner="TestProcessor")
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.HasOwner)
Example #13
0
def test_waiting_for_interval():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future
    timings.update_timestamp()
    timings.update_interval_timestamp()
    timings.interval = 10000

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False,
                                        ResourceStates.WaitingForInterval)
Example #14
0
class AnalyzerService(BaseService):
    """
    The Analyzer service is the process which will handle the analysis
    of resources (type Resource) and determine whether or not they
    warrant a request. Upon passing the necessary checks the service
    will send the resource to the next queue, or to the error queue.

    By this nature this service uses the BaseService class which
    within itself composities a greenlet.
    """
    def __init__(self, name, parent_logger=None, enable_service_recovery=False):
        BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
        self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
        self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
        self.queue = None

    def register(self):
        self.queue = self.get_directory_service_proxy().get_service("queue-service")

    def _analyze_resource(self, resource):
        # dirs = self.get_directory_service_proxy()
        can_request, possible_state = self.analyzer.can_request(resource)
        if can_request:
            size = self.queue.put_requests(resource)
            self.log.debug("resource put on request queue, size: [%d]" % size)
        else:  # can't request because something is wrong or not ready
            # Optimization: check the state and if the state is interval time related
            # put the resource on the appropriate frozen queue. This will prevent
            # a high resolution event loop from always evaluating resources that are
            # known to not be ready until the far future
            # If a resource cannot be requested it is either waiting for a time vector,
            # or is in an error state. Time in this case is a vector because it's is an
            # interval (interval delta, or reset window delta)
            if possible_state in [ResourceStates.WaitingForReset, ResourceStates.WaitingForInterval]:
                self.timing_sorter.sort(resource, possible_state, self.queue)
            elif possible_state in [ResourceStates.EdgeError, ResourceStates.Error, ResourceStates.HasOwner]:
                size = self.queue.put_analyze_error(resource)
                self.log.debug("resource put back on analyze queue, size: [%d]" % size)

    def event_loop(self):
        """
        The event loop.
        """

        while self.should_loop():
            # Don't do the below commented line, as the event loop will run fast
            # and will result in multiple lines being printed! Also that many of
            # log entries makes it confusing when narrowing down things. It is
            # better to tie a logging event to a logical event such as when a
            # resource may be requested (see below `can_request` method).
            # # don't do this -> self.log.debug("Size now: %d" % self.queue.analyzer_size())

            resource = self.queue.get_analyze()  # pop from queue

            if resource is not None:  # if an item exists
                self._analyze_resource(resource)

            gevent.idle()
Example #15
0
class FreezerService(BaseService):
    def __init__(self, name, parent_logger=None, enable_service_recovery=False):
        BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
        self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
        self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
        self.queue = None
        self.sleep_time = .05  # 50 (.05), 250 (.25), 500 (.5), 1000 (1)

    def register(self):
        self.queue = self.get_directory_service_proxy().get_service("queue-service")

    @abstractmethod
    def get_resource(self):
        """
        Use self.queue to get the resource in the implementation
        :return:
        """
        raise NotImplementedError("Please Implement this method")

    def _analyze_resource(self, resource):
        can_request, possible_state = self.analyzer.can_request(resource)

        if can_request:
            # size = self.put_resource(resource)
            size = self.queue.put_requests(resource)
            self.log.debug("resource put on request queue, size: [%d]" % size)
        else:
            if possible_state in [ResourceStates.WaitingForReset, ResourceStates.WaitingForInterval]:
                self.timing_sorter.sort(resource, possible_state, self.queue)
            elif possible_state in [ResourceStates.EdgeError, ResourceStates.Error, ResourceStates.HasOwner]:
                size = self.queue.put_analyze_error(resource)
                self.log.debug("resource put back on analyze queue, size: [%d]" % size)

    def event_loop(self):
        while self.should_loop():
            resource = self.get_resource()

            if resource is not None:  # if an item exists
                self._analyze_resource(resource)

            # In future revisions when I make use of real multi-threading this concept
            # will be realized. In terms of gevent, sleep is idle but with a greenlet waiting
            # for a scheduled time. So the below if still using gevent, does nothing but prevent
            # logging statements from running.
            gevent.sleep(self.sleep_time)  # in terms of gevent this is just a yield with a waiter
            gevent.idle()  # being a very good citizen, we yield again
Example #16
0
class AnalyzerService(BaseService):
    """
    The Analyzer service is the process which will handle the analysis
    of resources (type Resource) and determine whether or not they
    warrant a request. Upon passing the necessary checks the service
    will send the resource to the next queue, or to the error queue.

    By this nature this service uses the BaseService class which
    within itself composities a greenlet.
    """
    def __init__(self,
                 name,
                 parent_logger=None,
                 enable_service_recovery=False):
        BaseService.__init__(self,
                             name,
                             parent_logger=parent_logger,
                             enable_service_recovery=enable_service_recovery)
        self.analyzer = ResourceAnalyzer("resource-analyzer",
                                         parent_logger=self.log)
        self.timing_sorter = ResourceTimingSorter("timing-sorter",
                                                  parent_logger=self.log)
        self.queue = None

    def register(self):
        self.queue = self.get_directory_service_proxy().get_service(
            "queue-service")

    def _analyze_resource(self, resource):
        # dirs = self.get_directory_service_proxy()
        can_request, possible_state = self.analyzer.can_request(resource)
        if can_request:
            size = self.queue.put_requests(resource)
            self.log.debug("resource put on request queue, size: [%d]" % size)
        else:  # can't request because something is wrong or not ready
            # Optimization: check the state and if the state is interval time related
            # put the resource on the appropriate frozen queue. This will prevent
            # a high resolution event loop from always evaluating resources that are
            # known to not be ready until the far future
            # If a resource cannot be requested it is either waiting for a time vector,
            # or is in an error state. Time in this case is a vector because it's is an
            # interval (interval delta, or reset window delta)
            if possible_state in [
                    ResourceStates.WaitingForReset,
                    ResourceStates.WaitingForInterval
            ]:
                self.timing_sorter.sort(resource, possible_state, self.queue)
            elif possible_state in [
                    ResourceStates.EdgeError, ResourceStates.Error,
                    ResourceStates.HasOwner
            ]:
                size = self.queue.put_analyze_error(resource)
                self.log.debug(
                    "resource put back on analyze queue, size: [%d]" % size)

    def event_loop(self):
        """
        The event loop.
        """

        while self.should_loop():
            # Don't do the below commented line, as the event loop will run fast
            # and will result in multiple lines being printed! Also that many of
            # log entries makes it confusing when narrowing down things. It is
            # better to tie a logging event to a logical event such as when a
            # resource may be requested (see below `can_request` method).
            # # don't do this -> self.log.debug("Size now: %d" % self.queue.analyzer_size())

            resource = self.queue.get_analyze()  # pop from queue

            if resource is not None:  # if an item exists
                self._analyze_resource(resource)

            gevent.idle()
Example #17
0
 def __init__(self, name, parent_logger=None, enable_service_recovery=False):
     BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
     self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
     self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
     self.queue = None
     self.sleep_time = .05  # 50 (.05), 250 (.25), 500 (.5), 1000 (1)
Example #18
0
 def __init__(self, name, parent_logger=None, enable_service_recovery=False):
     BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
     self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
     self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
     self.queue = None