def test_resource_analyzer_edge_case():
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0  # limit reached
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.is_edge_case(resource)
    assert ra.can_request(resource) == (False, ResourceStates.EdgeError)
示例#2
0
def test_resource_analyzer_edge_case():
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0  # limit reached
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds(
    ) - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.is_edge_case(resource)
    assert ra.can_request(resource) == (False, ResourceStates.EdgeError)
def test_limit_reached_reset_infuture():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.WaitingForReset)
def test_resource_with_owner():
    """
    Scenario: resource has an owner.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders(), owner="TestProcessor")
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.HasOwner)
示例#5
0
def test_limit_reached_reset_infuture():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.WaitingForReset)
示例#6
0
def test_limit_reached_reset_ready():
    """
    Scenario: limit is 0, with time_to_reset passed
    Should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() - 2000  # in the past
    # timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (True, None)
def test_limit_reached_reset_ready():
    """
    Scenario: limit is 0, with time_to_reset passed
    Should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    timings.time_to_reset = timeutils.milliseconds() - 2000  # in the past
    # timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (True, None)
def test_with_limit_remaining():
    """
    Scenario: limit is 1000.
    should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds() - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource)
示例#9
0
def test_with_limit_remaining():
    """
    Scenario: limit is 1000.
    should be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds(
    ) - 1000  # was done after reset

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource)
示例#10
0
 def seed_data(self):
     uri = "mock://github/events-quick-interval"
     timings = ResourceTimings()
     resource = Resource(uri, timings, json=True)
     self.db.save_resource(resource)
     self.log.debug("data seeded, entry count: [%d]." %
                    self.db.resource_count())
示例#11
0
def test_resource_with_owner():
    """
    Scenario: resource has an owner.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() - 2000  # past now
    timings.last_request_timestamp = timeutils.milliseconds(
    ) - 1000  # was done after reset

    resource = Resource("mock://test",
                        timings,
                        ResourceHeaders(),
                        owner="TestProcessor")
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.HasOwner)
示例#12
0
def test_waiting_for_interval():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future
    timings.update_timestamp()
    timings.update_interval_timestamp()
    timings.interval = 10000

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False,
                                        ResourceStates.WaitingForInterval)
示例#13
0
def test_resource_is_new():
    timings = ResourceTimings()  # default timings
    resource = Resource("mock://test", timings, ResourceHeaders())
    resource.owner = "TESTER"

    assert resource.is_new()

    # updating the timings with a timestamp will now signify that it is
    # not a new resource, that it has been used and a request has been
    # called.
    resource.timings.update_timestamp()
    assert resource.is_new() is False
示例#14
0
def github_events_resource():
    token = 'c0698ac78b8f29412f9a358bacd2d34711cdf217'
    headers = {
        'User-Agent': 'CodeStats-Machine',
        'Authorization': "token %s" % token
    }

    timings = ResourceTimings()
    resource = Resource("https://api.github.com/events",
                        timings,
                        send_headers=headers)

    return resource
示例#15
0
def test_timings_reset_window():
    timings = ResourceTimings()  # default timings

    timings.time_to_reset = timeutils.milliseconds() + 1000  # one second or 1000 ms from now
    assert not timings.has_reset_window_past()

    timings.time_to_reset = timeutils.milliseconds() - 1000  # one second or 1000 ms ago
    assert timings.has_reset_window_past()
def test_waiting_for_interval():
    """
    Scenario: limit is 0, with time_to_reset in the future.
    Should not be able to request.
    """
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 1000
    timings.time_to_reset = timeutils.milliseconds() + 2000  # in the future
    timings.update_timestamp()
    timings.update_interval_timestamp()
    timings.interval = 10000

    resource = Resource("mock://test", timings, ResourceHeaders())
    ra = ResourceAnalyzer("test-resource-analyzer")

    assert ra.can_request(resource) == (False, ResourceStates.WaitingForInterval)
示例#17
0
def test_resource_timings():
    interval = 1000
    rate_limit = 1
    rate_limit_remaining = 1
    time_to_reset = 60
    etag = "ABCD"

    # default timings
    timings = ResourceTimings()

    # == assert defaults ==
    assert timings.interval == 1000
    assert timings.rate_limit == 1
    assert timings.rate_limit_remaining == 1
    assert timings.time_to_reset == 60
    assert timings.etag is None
    assert timings.last_request_timestamp is None

    # Update timestamp
    timings.update_timestamp()
    assert timings.last_request_timestamp is not None
    assert timings.last_request_timestamp > 0

    timings = ResourceTimings(interval, rate_limit, rate_limit_remaining,
                              time_to_reset, etag)

    # == asserts assignments ==
    assert timings.interval == interval
    assert timings.rate_limit == rate_limit
    assert timings.rate_limit_remaining == rate_limit_remaining
    assert timings.time_to_reset == time_to_reset
    assert timings.etag == etag

    # == assert methods ==
    old_timestamp = timings.last_request_timestamp
    timings.update_timestamp()
    assert timings.last_request_timestamp is not None
    assert timings.last_request_timestamp > old_timestamp
def test_resource_timing_sorter():
    mock_queue = MockQueueService("mock-queue")  # mock queue
    uri = "mock://github/events-quick-interval"  # create mock resource

    # create resource
    timings = ResourceTimings()
    resource = Resource(uri, timings, json=True)

    # get mock response
    session = mock_requests.create_mock_session()
    response = session.get(uri)

    resource.timings.update(response, resource.headers)

    # modify interval timing directly and make it trigger
    # in the future
    resource.timings.interval = 2000
    resource.timings.update_timestamp
    resource.timings.update_interval_timestamp()

    sorter = ResourceTimingSorter("test-sorter")
    sorter.sort(resource, ResourceStates.WaitingForInterval, mock_queue)

    def work():
        for n in range(25):
            sorter.sort(resource, ResourceStates.WaitingForInterval,
                        mock_queue)
            gevent.sleep(.10)

    work()

    # sometimes depending on the test runner or machine, things will be slow.
    # rather than assert on very specific timings, just ensure the counters
    # were hit
    assert mock_queue.f1000_calls >= 1
    assert mock_queue.f500_calls >= 1
    assert mock_queue.f250_calls >= 1
    assert mock_queue.f50_calls >= 1
    assert mock_queue.fa_calls >= 1
def test_response_parser():
    """
    This test will execute a mock request with mock results.
    :return:
    """
    uri = "mock://github/events/statustest"

    session = mock_requests.create_mock_session()
    response = session.get(uri)
    timings = ResourceTimings()
    resource = Resource(uri, timings, json=True)

    # parse out timings from headers
    # don't worry about getting the body to publish,
    # that is something the service should worry about
    parser = ResponseParser("response-parser")
    parser.parse(response, resource)

    # asserts new timings
    assert timings.interval == int(mock_requests.GLOBAL_MOCK_REQUEST_INTERVAL) * 1000
    assert timings.rate_limit == int(mock_requests.GLOBAL_MOCK_REQUEST_RATELIMIT)
    assert timings.rate_limit_remaining == int(mock_requests.GLOBAL_MOCK_REQUEST_REMAINING)
    assert timings.etag == mock_requests.GLOBAL_MOCK_REQUEST_ETAG1
    assert timings.time_to_reset == int(mock_requests.GLOBAL_MOCK_REQUEST_RESET) * 1000
示例#20
0
def test_timings_interval_past():
    timings = ResourceTimings()  # default timings

    # Update timestamp to 2000ms ago
    timings.last_request_timestamp = timeutils.milliseconds() - 2000

    # test that interval has not passed if set to 10000ms
    timings.interval = 10000
    timings.update_interval_timestamp()
    assert timings.has_interval_passed() is False

    # set interval to 1000, which when added to the timestamp should be less than now
    timings.interval = 1000
    timings.update_interval_timestamp()
    assert timings.has_interval_passed() is True

    # set the interval timestamp to now, this test can only be passed if the
    # interval PAST now, not equal
    now = timings.get_now()
    timings.interval_timestamp = now
    assert timings.has_interval_passed(now) is False

    # since the resolution is in milliseconds, one milliseconds is enough to ensure
    # the interval is passed
    timings.interval_timestamp = now - 1  # making it minus 1ms will make it past
    assert timings.has_interval_passed(now) is True
示例#21
0
def test_timings_requested_since_reset_window():
    timings = ResourceTimings()  # default timings

    timings.time_to_reset = timeutils.milliseconds() - 1000  # past
    timings.last_request_timestamp = timeutils.milliseconds() + 1000  # was done after reset
    assert timings.requested_since_reset()
示例#22
0
def test_timings_limit_reached():
    timings = ResourceTimings()  # default timings
    timings.rate_limit_remaining = 0
    assert timings.has_limit_been_reached()
示例#23
0
def test_resource_has_owner():
    timings = ResourceTimings()  # default timings
    resource = Resource("mock://test", timings, ResourceHeaders())
    resource.owner = "TESTER"

    assert resource.has_owner()