def test_the_instance_should_be_able_to_return_the_timespan_value_of_a_registered_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)
    assert rr.realm_timespan("TEST123") == 300

    rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_generate_a_redis_key_when_provided_with_a_realm():
    rr = RespectfulRequester()

    assert rr._realm_redis_key("TEST") == "%s:REALMS:TEST" % rr.redis_prefix
    assert rr._realm_redis_key("TEST2") == "%s:REALMS:TEST2" % rr.redis_prefix
    assert rr._realm_redis_key("TEST SPACED") == "%s:REALMS:TEST SPACED" % rr.redis_prefix
    assert rr._realm_redis_key("TEST ÉÉÉ") == "%s:REALMS:TEST ÉÉÉ" % rr.redis_prefix
Esempio n. 3
0
def test_the_instance_should_return_a_rate_limit_exception_if_the_request_is_not_allowed_on_one_or_multiple_registered_realms(
):
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=0, timespan=5)
    rr.register_realm("TEST234", max_requests=0, timespan=5)

    request_func = lambda: requests.get("http://google.com")

    with pytest.raises(RequestsRespectfulRateLimitedError):
        rr._perform_request(request_func, realms=["TEST123", "TEST234"])

    rr.update_realm("TEST123", max_requests=10)

    with pytest.raises(RequestsRespectfulRateLimitedError):
        rr._perform_request(request_func, realms=["TEST123", "TEST234"])

    rr.update_realm("TEST123", max_requests=0)
    rr.update_realm("TEST234", max_requests=10)

    with pytest.raises(RequestsRespectfulRateLimitedError):
        rr._perform_request(request_func, realms=["TEST123", "TEST234"])

    rr.unregister_realm("TEST123")
    rr.unregister_realm("TEST234")
Esempio n. 4
0
def test_the_instance_should_be_able_to_unregister_multiple_realms():
    rr = RespectfulRequester()

    realm_tuples = [
        ["TEST123", 100, 300],
        ["TEST234", 200, 600],
        ["TEST345", 300, 900],
    ]

    rr.register_realms(realm_tuples)

    request_func = lambda: requests.get("http://google.com")
    rr._perform_request(request_func, realms=["TEST123", "TEST234", "TEST345"])

    rr.unregister_realms(["TEST123", "TEST234", "TEST345"])

    assert rr.redis.get(rr._realm_redis_key("TEST123")) is None
    assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123")
    assert not len(
        rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST123")))

    assert rr.redis.get(rr._realm_redis_key("TEST234")) is None
    assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST234")
    assert not len(
        rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST234")))

    assert rr.redis.get(rr._realm_redis_key("TEST345")) is None
    assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST345")
    assert not len(
        rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST345")))
Esempio n. 5
0
def test_the_instance_should_ignore_invalid_realms_when_attempting_to_unregister(
):
    rr = RespectfulRequester()

    rr.unregister_realm("TEST123")
    rr.unregister_realm("TEST")
    rr.unregister_realm("TEST12345")
def test_the_instance_should_not_allow_a_request_to_be_made_on_an_unregistered_realm():
    rr = RespectfulRequester()

    request_func = lambda: requests.get("http://google.com")

    with pytest.raises(RequestsRespectfulError):
        rr.request(request_func, "TEST123")
Esempio n. 7
0
def test_the_instance_should_be_able_to_register_multiple_realms():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)

    realm_tuples = [
        ["TEST123", 100, 300],
        ["TEST234", 200, 600],
        ["TEST345", 300, 900],
    ]

    rr.register_realms(realm_tuples)

    assert rr.realm_max_requests("TEST123") == 100
    assert rr.realm_timespan("TEST123") == 300
    assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123")

    assert rr.realm_max_requests("TEST234") == 200
    assert rr.realm_timespan("TEST234") == 600
    assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST234")

    assert rr.realm_max_requests("TEST345") == 300
    assert rr.realm_timespan("TEST345") == 900
    assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST345")

    rr.unregister_realm("TEST123")
    rr.unregister_realm("TEST234")
    rr.unregister_realm("TEST345")
Esempio n. 8
0
def test_the_instance_should_be_able_to_fetch_a_list_of_registered_realms():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)

    assert "TEST123" in rr.fetch_registered_realms()

    rr.unregister_realm("TEST123")
def test_the_instance_should_validate_that_the_request_lambda_is_actually_a_requests_call():
    rr = RespectfulRequester()

    with pytest.raises(RequestsRespectfulError):
        rr._validate_request_func(lambda: 1 + 1)

    rr._validate_request_func(lambda: requests.get("http://google.com"))
    rr._validate_request_func(lambda: getattr(requests, "get")("http://google.com"))
def test_the_instance_should_perform_the_request_if_it_is_allowed_to_on_a_registered_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=1000, timespan=5)

    request_func = lambda: requests.get("http://google.com")
    assert type(rr._perform_request(request_func, "TEST123")) == requests.Response

    rr.unregister_realm("TEST123")
Esempio n. 11
0
def test_the_instance_should_be_able_to_update_a_registered_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)
    rr.update_realm("TEST123", max_requests=1000, timespan=3000)

    assert rr.realm_max_requests("TEST123") == 1000
    assert rr.realm_timespan("TEST123") == 3000

    rr.unregister_realm("TEST123")
Esempio n. 12
0
def test_the_instance_should_not_overwrite_when_registering_a_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)
    rr.register_realm("TEST123", max_requests=1000, timespan=3000)

    assert rr.realm_max_requests("TEST123") == 100
    assert rr.realm_timespan("TEST123") == 300

    rr.unregister_realm("TEST123")
def test_the_requests_module_name_configuration_value_should_have_the_expected_effect():
    rr = RespectfulRequester()

    RespectfulRequester.configure(requests_module_name="r")

    request_func = lambda: r.get("http://google.com")

    rr._validate_request_func(request_func)

    RespectfulRequester.configure_default()
Esempio n. 14
0
def test_the_instance_should_be_able_to_register_a_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)

    assert rr.realm_max_requests("TEST123") == 100
    assert rr.realm_timespan("TEST123") == 300
    assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123")

    rr.unregister_realm("TEST123")
Esempio n. 15
0
def test_the_instance_should_ignore_invalid_values_when_updating_a_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)
    rr.update_realm("TEST123", max_requests="FOO", timespan="BAR", fake=True)

    assert rr.realm_max_requests("TEST123") == 100
    assert rr.realm_timespan("TEST123") == 300

    rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_determine_if_it_can_perform_a_request_for_a_registered_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=1000, timespan=5)

    assert rr._can_perform_request("TEST123")

    rr.update_realm("TEST123", max_requests=0)

    assert not rr._can_perform_request("TEST123")

    rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_fetch_the_information_of_a_registered_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)

    assert b"max_requests" in rr._fetch_realm_info("TEST123")
    assert rr._fetch_realm_info("TEST123")[b"max_requests"] == b"100"

    assert b"timespan" in rr._fetch_realm_info("TEST123")
    assert rr._fetch_realm_info("TEST123")[b"timespan"] == b"300"

    rr.unregister_realm("TEST123")
def test_the_instance_should_get_the_same_results_by_using_the_requests_proxy_as_when_using_the_request_method():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)

    assert type(rr.get("http://google.com", realm="TEST123")) == requests.Response

    rr.update_realm("TEST123", max_requests=0)

    with pytest.raises(RequestsRespectfulRateLimitedError):
        rr.get("http://google.com", realm="TEST123")

    rr.unregister_realm("TEST123")
Esempio n. 19
0
def test_the_instance_should_recognize_the_requests_proxy_methods():
    rr = RespectfulRequester()

    getattr(rr, "delete")
    getattr(rr, "get")
    getattr(rr, "head")
    getattr(rr, "options")
    getattr(rr, "patch")
    getattr(rr, "post")
    getattr(rr, "put")

    with pytest.raises(AttributeError):
        getattr(rr, "foo")
def test_the_class_should_be_able_to_restore_the_default_configuration_values():
    rr = RespectfulRequester()

    RespectfulRequester.configure(safety_threshold=20)
    RespectfulRequester.configure(requests_module_name="r")
    RespectfulRequester.configure(redis={"host": "0.0.0.0", "port": 6379, "database": 1})

    RespectfulRequester.configure_default()

    assert rr._config()["safety_threshold"] == 10
    assert rr._config()["requests_module_name"] == "requests"
    assert rr._config()["redis"]["host"] == "localhost"
    assert rr._config()["redis"]["database"] == 0
def test_the_instance_should_be_able_to_unregister_a_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=100, timespan=300)

    request_func = lambda: requests.get("http://google.com")
    rr._perform_request(request_func, "TEST123")

    rr.unregister_realm("TEST123")

    assert rr.redis.get(rr._realm_redis_key("TEST123")) is None
    assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123")
    assert not len(rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST123")))
def test_the_instance_should_be_able_to_wait_for_a_request_to_be_allowed_on_a_registered_realm():
    rr = RespectfulRequester()

    RespectfulRequester.configure(safety_threshold=0)

    rr.register_realm("TEST123", max_requests=1, timespan=2)

    request_func = lambda: requests.get("http://google.com")

    rr.request(request_func, "TEST123", wait=True)
    rr.request(request_func, "TEST123", wait=True)
    rr.request(request_func, "TEST123", wait=True)

    rr.unregister_realm("TEST123")

    RespectfulRequester.configure_default()
def test_the_instance_should_be_able_to_determine_the_amount_of_requests_performed_in_a_timespan_for_a_registered_realm():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=1000, timespan=5)

    assert rr._requests_in_timespan("TEST123") == 0

    request_func = lambda: requests.get("http://google.com")

    rr._perform_request(request_func, "TEST123")
    rr._perform_request(request_func, "TEST123")
    rr._perform_request(request_func, "TEST123")

    assert rr._requests_in_timespan("TEST123") == 3

    rr.unregister_realm("TEST123")
def test_the_safety_threshold_configuration_value_should_have_the_expected_effect():
    rr = RespectfulRequester()

    rr.register_realm("TEST123", max_requests=11, timespan=300)

    RespectfulRequester.configure(safety_threshold=10)

    request_func = lambda: requests.get("http://google.com")

    rr.request(request_func, "TEST123")

    with pytest.raises(RequestsRespectfulRateLimitedError):
        rr.request(request_func, "TEST123")

    RespectfulRequester.configure_default()

    rr.unregister_realm("TEST123")
Esempio n. 25
0
 def __init__(self):
     self.rr = RespectfulRequester()
     self.rr.register_realm("sochain", max_requests=299, timespan=60)
     self.requestCount = 0
Esempio n. 26
0
def test_setup():
    rr = RespectfulRequester()
    rr.unregister_realm("TEST123")

    RespectfulRequester.configure_default()
Esempio n. 27
0
def test_the_instance_should_have_a_property_that_holds_a_redis_object():
    rr = RespectfulRequester()
    assert type(rr.redis) == redis.StrictRedis
Esempio n. 28
0
def test_the_instance_should_have_a_property_that_holds_a_redis_prefix():
    rr = RespectfulRequester()
    assert rr.redis_prefix == "RespectfulRequester"
Esempio n. 29
0
# Requests Respectful (rate limiting, waiting)
if REMOTE is True:
    from urllib.parse import urlparse
    url_object = urlparse(os.getenv('REDIS_URL'))
    logger.info('Connecting to redis at %s:%s', url_object.hostname,
                url_object.port)
    RespectfulRequester.configure(redis={
        "host": url_object.hostname,
        "port": url_object.port,
        "password": url_object.password,
        "database": 0
    },
                                  safety_threshold=5)

# This creates a Realm called "source" that allows 60 requests per minute maximum.
rr = RespectfulRequester()
rr.register_realm("Source", max_requests=60, timespan=60)

# Applications installed
INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Local apps. Update these if you add or change app names!
    'datauploader.apps.DatauploaderConfig',
    'open_humans.apps.OpenHumansConfig',
    'main.apps.MainConfig'
Esempio n. 30
0
def test_the_instance_should_be_able_to_access_the_global_config():
    rr = RespectfulRequester()

    assert "redis" in rr._config()
    assert "safety_threshold" in rr._config()
    assert "requests_module_name" in rr._config()