def test_resource_analyzer_edge_case():
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 0 # limit reached
timings.time_to_reset = timeutils.milliseconds() - 2000 # past now
timings.last_request_timestamp = timeutils.milliseconds() - 1000 # was done after reset
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.is_edge_case(resource)
assert ra.can_request(resource) == (False, ResourceStates.EdgeError)
def test_resource_analyzer_edge_case():
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 0 # limit reached
timings.time_to_reset = timeutils.milliseconds() - 2000 # past now
timings.last_request_timestamp = timeutils.milliseconds(
) - 1000 # was done after reset
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.is_edge_case(resource)
assert ra.can_request(resource) == (False, ResourceStates.EdgeError)
def test_limit_reached_reset_infuture():
"""
Scenario: limit is 0, with time_to_reset in the future.
Should not be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 0
timings.time_to_reset = timeutils.milliseconds() + 2000 # in the future
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (False, ResourceStates.WaitingForReset)
def test_limit_reached_reset_infuture():
"""
Scenario: limit is 0, with time_to_reset in the future.
Should not be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 0
timings.time_to_reset = timeutils.milliseconds() + 2000 # in the future
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (False, ResourceStates.WaitingForReset)
def test_resource_with_owner():
"""
Scenario: resource has an owner.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 1000
timings.time_to_reset = timeutils.milliseconds() - 2000 # past now
timings.last_request_timestamp = timeutils.milliseconds() - 1000 # was done after reset
resource = Resource("mock://test", timings, ResourceHeaders(), owner="TestProcessor")
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (False, ResourceStates.HasOwner)
def __init__(self,
name,
parent_logger=None,
enable_service_recovery=False):
BaseService.__init__(self,
name,
parent_logger=parent_logger,
enable_service_recovery=enable_service_recovery)
self.analyzer = ResourceAnalyzer("resource-analyzer",
parent_logger=self.log)
self.timing_sorter = ResourceTimingSorter("timing-sorter",
parent_logger=self.log)
self.queue = None
def test_limit_reached_reset_ready():
"""
Scenario: limit is 0, with time_to_reset passed
Should be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 0
timings.time_to_reset = timeutils.milliseconds() - 2000 # in the past
# timings.last_request_timestamp = timeutils.milliseconds() - 1000 # was done after reset
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (True, None)
def test_limit_reached_reset_ready():
"""
Scenario: limit is 0, with time_to_reset passed
Should be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 0
timings.time_to_reset = timeutils.milliseconds() - 2000 # in the past
# timings.last_request_timestamp = timeutils.milliseconds() - 1000 # was done after reset
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (True, None)
def test_with_limit_remaining():
"""
Scenario: limit is 1000.
should be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 1000
timings.time_to_reset = timeutils.milliseconds() - 2000 # past now
timings.last_request_timestamp = timeutils.milliseconds() - 1000 # was done after reset
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource)
def test_with_limit_remaining():
"""
Scenario: limit is 1000.
should be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 1000
timings.time_to_reset = timeutils.milliseconds() - 2000 # past now
timings.last_request_timestamp = timeutils.milliseconds(
) - 1000 # was done after reset
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource)
def test_waiting_for_interval():
"""
Scenario: limit is 0, with time_to_reset in the future.
Should not be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 1000
timings.time_to_reset = timeutils.milliseconds() + 2000 # in the future
timings.update_timestamp()
timings.update_interval_timestamp()
timings.interval = 10000
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (False, ResourceStates.WaitingForInterval)
def test_resource_with_owner():
"""
Scenario: resource has an owner.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 1000
timings.time_to_reset = timeutils.milliseconds() - 2000 # past now
timings.last_request_timestamp = timeutils.milliseconds(
) - 1000 # was done after reset
resource = Resource("mock://test",
timings,
ResourceHeaders(),
owner="TestProcessor")
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (False, ResourceStates.HasOwner)
def test_waiting_for_interval():
"""
Scenario: limit is 0, with time_to_reset in the future.
Should not be able to request.
"""
timings = ResourceTimings() # default timings
timings.rate_limit_remaining = 1000
timings.time_to_reset = timeutils.milliseconds() + 2000 # in the future
timings.update_timestamp()
timings.update_interval_timestamp()
timings.interval = 10000
resource = Resource("mock://test", timings, ResourceHeaders())
ra = ResourceAnalyzer("test-resource-analyzer")
assert ra.can_request(resource) == (False,
ResourceStates.WaitingForInterval)
class AnalyzerService(BaseService):
"""
The Analyzer service is the process which will handle the analysis
of resources (type Resource) and determine whether or not they
warrant a request. Upon passing the necessary checks the service
will send the resource to the next queue, or to the error queue.
By this nature this service uses the BaseService class which
within itself composities a greenlet.
"""
def __init__(self, name, parent_logger=None, enable_service_recovery=False):
BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
self.queue = None
def register(self):
self.queue = self.get_directory_service_proxy().get_service("queue-service")
def _analyze_resource(self, resource):
# dirs = self.get_directory_service_proxy()
can_request, possible_state = self.analyzer.can_request(resource)
if can_request:
size = self.queue.put_requests(resource)
self.log.debug("resource put on request queue, size: [%d]" % size)
else: # can't request because something is wrong or not ready
# Optimization: check the state and if the state is interval time related
# put the resource on the appropriate frozen queue. This will prevent
# a high resolution event loop from always evaluating resources that are
# known to not be ready until the far future
# If a resource cannot be requested it is either waiting for a time vector,
# or is in an error state. Time in this case is a vector because it's is an
# interval (interval delta, or reset window delta)
if possible_state in [ResourceStates.WaitingForReset, ResourceStates.WaitingForInterval]:
self.timing_sorter.sort(resource, possible_state, self.queue)
elif possible_state in [ResourceStates.EdgeError, ResourceStates.Error, ResourceStates.HasOwner]:
size = self.queue.put_analyze_error(resource)
self.log.debug("resource put back on analyze queue, size: [%d]" % size)
def event_loop(self):
"""
The event loop.
"""
while self.should_loop():
# Don't do the below commented line, as the event loop will run fast
# and will result in multiple lines being printed! Also that many of
# log entries makes it confusing when narrowing down things. It is
# better to tie a logging event to a logical event such as when a
# resource may be requested (see below `can_request` method).
# # don't do this -> self.log.debug("Size now: %d" % self.queue.analyzer_size())
resource = self.queue.get_analyze() # pop from queue
if resource is not None: # if an item exists
self._analyze_resource(resource)
gevent.idle()
class FreezerService(BaseService):
def __init__(self, name, parent_logger=None, enable_service_recovery=False):
BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
self.queue = None
self.sleep_time = .05 # 50 (.05), 250 (.25), 500 (.5), 1000 (1)
def register(self):
self.queue = self.get_directory_service_proxy().get_service("queue-service")
@abstractmethod
def get_resource(self):
"""
Use self.queue to get the resource in the implementation
:return:
"""
raise NotImplementedError("Please Implement this method")
def _analyze_resource(self, resource):
can_request, possible_state = self.analyzer.can_request(resource)
if can_request:
# size = self.put_resource(resource)
size = self.queue.put_requests(resource)
self.log.debug("resource put on request queue, size: [%d]" % size)
else:
if possible_state in [ResourceStates.WaitingForReset, ResourceStates.WaitingForInterval]:
self.timing_sorter.sort(resource, possible_state, self.queue)
elif possible_state in [ResourceStates.EdgeError, ResourceStates.Error, ResourceStates.HasOwner]:
size = self.queue.put_analyze_error(resource)
self.log.debug("resource put back on analyze queue, size: [%d]" % size)
def event_loop(self):
while self.should_loop():
resource = self.get_resource()
if resource is not None: # if an item exists
self._analyze_resource(resource)
# In future revisions when I make use of real multi-threading this concept
# will be realized. In terms of gevent, sleep is idle but with a greenlet waiting
# for a scheduled time. So the below if still using gevent, does nothing but prevent
# logging statements from running.
gevent.sleep(self.sleep_time) # in terms of gevent this is just a yield with a waiter
gevent.idle() # being a very good citizen, we yield again
class AnalyzerService(BaseService):
"""
The Analyzer service is the process which will handle the analysis
of resources (type Resource) and determine whether or not they
warrant a request. Upon passing the necessary checks the service
will send the resource to the next queue, or to the error queue.
By this nature this service uses the BaseService class which
within itself composities a greenlet.
"""
def __init__(self,
name,
parent_logger=None,
enable_service_recovery=False):
BaseService.__init__(self,
name,
parent_logger=parent_logger,
enable_service_recovery=enable_service_recovery)
self.analyzer = ResourceAnalyzer("resource-analyzer",
parent_logger=self.log)
self.timing_sorter = ResourceTimingSorter("timing-sorter",
parent_logger=self.log)
self.queue = None
def register(self):
self.queue = self.get_directory_service_proxy().get_service(
"queue-service")
def _analyze_resource(self, resource):
# dirs = self.get_directory_service_proxy()
can_request, possible_state = self.analyzer.can_request(resource)
if can_request:
size = self.queue.put_requests(resource)
self.log.debug("resource put on request queue, size: [%d]" % size)
else: # can't request because something is wrong or not ready
# Optimization: check the state and if the state is interval time related
# put the resource on the appropriate frozen queue. This will prevent
# a high resolution event loop from always evaluating resources that are
# known to not be ready until the far future
# If a resource cannot be requested it is either waiting for a time vector,
# or is in an error state. Time in this case is a vector because it's is an
# interval (interval delta, or reset window delta)
if possible_state in [
ResourceStates.WaitingForReset,
ResourceStates.WaitingForInterval
]:
self.timing_sorter.sort(resource, possible_state, self.queue)
elif possible_state in [
ResourceStates.EdgeError, ResourceStates.Error,
ResourceStates.HasOwner
]:
size = self.queue.put_analyze_error(resource)
self.log.debug(
"resource put back on analyze queue, size: [%d]" % size)
def event_loop(self):
"""
The event loop.
"""
while self.should_loop():
# Don't do the below commented line, as the event loop will run fast
# and will result in multiple lines being printed! Also that many of
# log entries makes it confusing when narrowing down things. It is
# better to tie a logging event to a logical event such as when a
# resource may be requested (see below `can_request` method).
# # don't do this -> self.log.debug("Size now: %d" % self.queue.analyzer_size())
resource = self.queue.get_analyze() # pop from queue
if resource is not None: # if an item exists
self._analyze_resource(resource)
gevent.idle()
def __init__(self, name, parent_logger=None, enable_service_recovery=False):
BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
self.queue = None
self.sleep_time = .05 # 50 (.05), 250 (.25), 500 (.5), 1000 (1)
def __init__(self, name, parent_logger=None, enable_service_recovery=False):
BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
self.analyzer = ResourceAnalyzer("resource-analyzer", parent_logger=self.log)
self.timing_sorter = ResourceTimingSorter("timing-sorter", parent_logger=self.log)
self.queue = None