def test_the_instance_should_not_allow_a_request_to_be_made_on_an_unregistered_realm(): rr = RespectfulRequester() request_func = lambda: requests.get("http://google.com") with pytest.raises(RequestsRespectfulError): rr.request(request_func, "TEST123")
def test_the_instance_should_be_able_to_unregister_multiple_realms(): rr = RespectfulRequester() realm_tuples = [ ["TEST123", 100, 300], ["TEST234", 200, 600], ["TEST345", 300, 900], ] rr.register_realms(realm_tuples) request_func = lambda: requests.get("http://google.com") rr._perform_request(request_func, realms=["TEST123", "TEST234", "TEST345"]) rr.unregister_realms(["TEST123", "TEST234", "TEST345"]) assert rr.redis.get(rr._realm_redis_key("TEST123")) is None assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123") assert not len(rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST123"))) assert rr.redis.get(rr._realm_redis_key("TEST234")) is None assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST234") assert not len(rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST234"))) assert rr.redis.get(rr._realm_redis_key("TEST345")) is None assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST345") assert not len(rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST345")))
def test_the_instance_should_not_overwrite_when_registering_a_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) rr.register_realm("TEST123", max_requests=1000, timespan=3000) assert rr.realm_max_requests("TEST123") == 100 assert rr.realm_timespan("TEST123") == 300 rr.unregister_realm("TEST123")
def test_the_instance_should_ignore_invalid_values_when_updating_a_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) rr.update_realm("TEST123", max_requests="FOO", timespan="BAR", fake=True) assert rr.realm_max_requests("TEST123") == 100 assert rr.realm_timespan("TEST123") == 300 rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_update_a_registered_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) rr.update_realm("TEST123", max_requests=1000, timespan=3000) assert rr.realm_max_requests("TEST123") == 1000 assert rr.realm_timespan("TEST123") == 3000 rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_fetch_the_information_of_a_registered_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) assert b"max_requests" in rr._fetch_realm_info("TEST123") assert rr._fetch_realm_info("TEST123")[b"max_requests"] == b"100" assert b"timespan" in rr._fetch_realm_info("TEST123") assert rr._fetch_realm_info("TEST123")[b"timespan"] == b"300" rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_determine_if_it_can_perform_a_request_for_a_registered_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=1000, timespan=5) assert rr._can_perform_request("TEST123") rr.update_realm("TEST123", max_requests=0) assert not rr._can_perform_request("TEST123") rr.unregister_realm("TEST123")
def test_the_instance_should_get_the_same_results_by_using_the_requests_proxy_as_when_using_the_request_method(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) assert type(rr.get("http://google.com", realm="TEST123")) == requests.Response rr.update_realm("TEST123", max_requests=0) with pytest.raises(RequestsRespectfulRateLimitedError): rr.get("http://google.com", realm="TEST123") rr.unregister_realm("TEST123")
def test_the_instance_should_ignore_invalid_realms_when_attempting_to_unregister( ): rr = RespectfulRequester() rr.unregister_realm("TEST123") rr.unregister_realm("TEST") rr.unregister_realm("TEST12345")
def test_the_instance_should_validate_that_the_request_lambda_is_actually_a_requests_call(): rr = RespectfulRequester() with pytest.raises(RequestsRespectfulError): rr._validate_request_func(lambda: 1 + 1) rr._validate_request_func(lambda: requests.get("http://google.com")) rr._validate_request_func(lambda: getattr(requests, "get")("http://google.com"))
def test_the_instance_should_be_able_to_unregister_a_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) request_func = lambda: requests.get("http://google.com") rr._perform_request(request_func, "TEST123") rr.unregister_realm("TEST123") assert rr.redis.get(rr._realm_redis_key("TEST123")) is None assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123") assert not len(rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST123")))
def test_the_requests_module_name_configuration_value_should_have_the_expected_effect(): rr = RespectfulRequester() RespectfulRequester.configure(requests_module_name="r") request_func = lambda: r.get("http://google.com") rr._validate_request_func(request_func) RespectfulRequester.configure_default()
def test_the_instance_should_be_able_to_register_a_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) assert rr.realm_max_requests("TEST123") == 100 assert rr.realm_timespan("TEST123") == 300 assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123") rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_return_the_timespan_value_of_a_registered_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) assert rr.realm_timespan("TEST123") == 300 rr.unregister_realm("TEST123")
def test_the_instance_should_return_a_rate_limit_exception_if_the_request_is_not_allowed_on_a_registered_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=0, timespan=5) request_func = lambda: requests.get("http://google.com") with pytest.raises(RequestsRespectfulRateLimitedError): rr._perform_request(request_func, "TEST123") rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_fetch_a_list_of_registered_realms(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) assert "TEST123" in rr.fetch_registered_realms() rr.unregister_realm("TEST123")
def test_the_instance_should_perform_the_request_if_it_is_allowed_to_on_a_registered_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=1000, timespan=5) request_func = lambda: requests.get("http://google.com") assert type(rr._perform_request(request_func, "TEST123")) == requests.Response rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_generate_a_redis_key_when_provided_with_a_realm(): rr = RespectfulRequester() assert rr._realm_redis_key("TEST") == "%s:REALMS:TEST" % rr.redis_prefix assert rr._realm_redis_key("TEST2") == "%s:REALMS:TEST2" % rr.redis_prefix assert rr._realm_redis_key("TEST SPACED") == "%s:REALMS:TEST SPACED" % rr.redis_prefix assert rr._realm_redis_key("TEST ÉÉÉ") == "%s:REALMS:TEST ÉÉÉ" % rr.redis_prefix
def test_the_instance_should_recognize_the_requests_proxy_methods(): rr = RespectfulRequester() getattr(rr, "delete") getattr(rr, "get") getattr(rr, "head") getattr(rr, "options") getattr(rr, "patch") getattr(rr, "post") getattr(rr, "put") with pytest.raises(AttributeError): getattr(rr, "foo")
class Requester: def __init__(self): self.rr = RespectfulRequester() self.rr.register_realm("sochain", max_requests=299, timespan=60) self.requestCount = 0 def get_spent_transactions(self, address, after=''): try: print("Request #" + str(self.requestCount) + " | Data on address " + address) response = self.rr.get(API_URL + "get_tx_spent/BTC/" + address + '/' + after, realms=["sochain"], wait=True) txs = response.json()['data']['txs'] print("Request #" + str(self.requestCount) + " | Got " + str(len(txs)) + " transactions") except: print("Request #" + str(self.requestCount) + " | Got error") txs = [] self.requestCount += 1 return txs def get_transaction_data(self, txid): try: print("Request #" + str(self.requestCount) + " | Data on transaction " + txid) response = self.rr.get(API_URL + "get_tx_outputs/BTC/" + txid, realms=["sochain"], wait=True) outputs = response.json()['data']['outputs'] print("Request #" + str(self.requestCount) + " | Got " + str(len(outputs)) + " outputs") except: print("Request #" + str(self.requestCount) + " | Got error") outputs = [] self.requestCount += 1 return outputs
def test_the_instance_should_be_able_to_determine_the_amount_of_requests_performed_in_a_timespan_for_a_registered_realm( ): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=1000, timespan=5) assert rr._requests_in_timespan("TEST123") == 0 request_func = lambda: requests.get("http://google.com") rr._perform_request(request_func, realms=["TEST123"]) rr._perform_request(request_func, realms=["TEST123"]) rr._perform_request(request_func, realms=["TEST123"]) assert rr._requests_in_timespan("TEST123") == 3 rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_wait_for_a_request_to_be_allowed_on_a_registered_realm(): rr = RespectfulRequester() RespectfulRequester.configure(safety_threshold=0) rr.register_realm("TEST123", max_requests=1, timespan=2) request_func = lambda: requests.get("http://google.com") rr.request(request_func, "TEST123", wait=True) rr.request(request_func, "TEST123", wait=True) rr.request(request_func, "TEST123", wait=True) rr.unregister_realm("TEST123") RespectfulRequester.configure_default()
def test_the_instance_should_be_able_to_unregister_multiple_realms(): rr = RespectfulRequester() realm_tuples = [ ["TEST123", 100, 300], ["TEST234", 200, 600], ["TEST345", 300, 900], ] rr.register_realms(realm_tuples) request_func = lambda: requests.get("http://google.com") rr._perform_request(request_func, realms=["TEST123", "TEST234", "TEST345"]) rr.unregister_realms(["TEST123", "TEST234", "TEST345"]) assert rr.redis.get(rr._realm_redis_key("TEST123")) is None assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123") assert not len( rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST123"))) assert rr.redis.get(rr._realm_redis_key("TEST234")) is None assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST234") assert not len( rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST234"))) assert rr.redis.get(rr._realm_redis_key("TEST345")) is None assert not rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST345") assert not len( rr.redis.keys("%s:REQUESTS:%s:*" % (rr.redis_prefix, "TEST345")))
def test_setup(): rr = RespectfulRequester() rr.unregister_realm("TEST123") RespectfulRequester.configure_default()
def test_the_safety_threshold_configuration_value_should_have_the_expected_effect( ): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=11, timespan=300) RespectfulRequester.configure(safety_threshold=10) request_func = lambda: requests.get("http://google.com") rr.request(request_func, realms=["TEST123"]) with pytest.raises(RequestsRespectfulRateLimitedError): rr.request(request_func, realms=["TEST123"]) RespectfulRequester.configure_default() rr.unregister_realm("TEST123")
def test_the_instance_should_ignore_invalid_realms_when_attempting_to_unregister(): rr = RespectfulRequester() rr.unregister_realm("TEST123") rr.unregister_realm("TEST") rr.unregister_realm("TEST12345")
def test_the_instance_should_have_a_property_that_holds_a_redis_object(): rr = RespectfulRequester() assert type(rr.redis) == redis.StrictRedis
def test_the_class_should_validate_provided_configuration_values(): with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(redis="REDIS") with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(redis=dict()) with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(redis={ "host": "localhost", "port": 6379 }) RespectfulRequester.configure(redis={ "host": "localhost", "port": 6379, "database": 0 }) with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(safety_threshold="SAFETY") with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(safety_threshold=-15) RespectfulRequester.configure(safety_threshold=10) with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(requests_module_name=100) RespectfulRequester.configure(requests_module_name="requests") RespectfulRequester.configure_default()
def test_the_instance_should_be_able_to_access_the_global_config(): rr = RespectfulRequester() assert "redis" in rr._config() assert "safety_threshold" in rr._config() assert "requests_module_name" in rr._config()
} # Configure Django App for Heroku. if ON_HEROKU: django_heroku.settings(locals()) REMOTE = os.getenv("REMOTE") if REMOTE: from urllib.parse import urlparse url_object = urlparse(os.getenv("REDIS_URL")) RespectfulRequester.configure( redis={ "host": url_object.hostname, "port": url_object.port, "password": url_object.password, "database": 0, }, safety_threshold=5, ) SECURE_SSL_REDIRECT = True SESSION_COOKIE_SECURE = True CSRF_COOKIE_SECURE = True import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.redis import RedisIntegration sentry_sdk.init( dsn=os.getenv("SENTRY_DSN"),
def test_the_instance_should_be_able_to_wait_for_a_request_to_be_allowed_on_multiple_registered_realms( ): rr = RespectfulRequester() RespectfulRequester.configure(safety_threshold=0) rr.register_realm("TEST123", max_requests=1, timespan=5) rr.register_realm("TEST234", max_requests=1, timespan=2) request_func = lambda: requests.get("http://google.com") rr.request(request_func, realms=["TEST123", "TEST234"], wait=True) rr.request(request_func, realms=["TEST123", "TEST234"], wait=True) rr.request(request_func, realms=["TEST123", "TEST234"], wait=True) rr.unregister_realm("TEST123") rr.unregister_realm("TEST234") RespectfulRequester.configure_default()
def test_the_instance_should_be_able_to_register_multiple_realms(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) realm_tuples = [ ["TEST123", 100, 300], ["TEST234", 200, 600], ["TEST345", 300, 900], ] rr.register_realms(realm_tuples) assert rr.realm_max_requests("TEST123") == 100 assert rr.realm_timespan("TEST123") == 300 assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST123") assert rr.realm_max_requests("TEST234") == 200 assert rr.realm_timespan("TEST234") == 600 assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST234") assert rr.realm_max_requests("TEST345") == 300 assert rr.realm_timespan("TEST345") == 900 assert rr.redis.sismember("%s:REALMS" % rr.redis_prefix, "TEST345") rr.unregister_realm("TEST123") rr.unregister_realm("TEST234") rr.unregister_realm("TEST345")
def test_the_instance_should_get_the_same_results_by_using_the_requests_proxy_as_when_using_the_request_method( ): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=100, timespan=300) assert type(rr.get("http://google.com", realm="TEST123")) == requests.Response rr.update_realm("TEST123", max_requests=0) with pytest.raises(RequestsRespectfulRateLimitedError): rr.get("http://google.com", realm="TEST123") rr.unregister_realm("TEST123")
def test_the_instance_should_be_able_to_determine_the_amount_of_requests_performed_in_a_timespan_for_a_registered_realm(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=1000, timespan=5) assert rr._requests_in_timespan("TEST123") == 0 request_func = lambda: requests.get("http://google.com") rr._perform_request(request_func, "TEST123") rr._perform_request(request_func, "TEST123") rr._perform_request(request_func, "TEST123") assert rr._requests_in_timespan("TEST123") == 3 rr.unregister_realm("TEST123")
def test_the_class_should_be_able_to_restore_the_default_configuration_values( ): rr = RespectfulRequester() RespectfulRequester.configure(safety_threshold=20) RespectfulRequester.configure(requests_module_name="r") RespectfulRequester.configure(redis={ "host": "0.0.0.0", "port": 6379, "database": 1 }) RespectfulRequester.configure_default() assert rr._config()["safety_threshold"] == 10 assert rr._config()["requests_module_name"] == "requests" assert rr._config()["redis"]["host"] == "localhost" assert rr._config()["redis"]["database"] == 0
def test_the_safety_threshold_configuration_value_should_have_the_expected_effect(): rr = RespectfulRequester() rr.register_realm("TEST123", max_requests=11, timespan=300) RespectfulRequester.configure(safety_threshold=10) request_func = lambda: requests.get("http://google.com") rr.request(request_func, "TEST123") with pytest.raises(RequestsRespectfulRateLimitedError): rr.request(request_func, "TEST123") RespectfulRequester.configure_default() rr.unregister_realm("TEST123")
db = init_db() else: from models import db # noqa redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379') url_object = urlparse.urlparse(redis_url) logger.info('Connecting to redis at %s:%s', url_object.hostname, url_object.port) RespectfulRequester.configure( redis={ 'host': url_object.hostname, 'port': url_object.port, 'password': url_object.password, 'database': 0, }, safety_threshold=5) requests = RespectfulRequester() requests.register_realm('fitbit', max_requests=3600, timespan=3600) requests.update_realm('fitbit', max_requests=3600, timespan=3600) # Only use cached data if cached less than CACHE_MAX before now. # Cache target dates older than CACHE_MIN before now. # Based on these settings 23 URLs won't be cached, and full data retrieval # requires around 6 to 7 hours per year of data. CACHE_MAX = timedelta(weeks=1) CACHE_MIN = timedelta(days=1)
def test_the_class_should_validate_provided_configuration_values(): with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(redis="REDIS") with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(redis=dict()) with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(redis={"host": "localhost", "port": 6379}) RespectfulRequester.configure(redis={"host": "localhost", "port": 6379, "database": 0}) with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(safety_threshold="SAFETY") with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(safety_threshold=-15) RespectfulRequester.configure(safety_threshold=10) with pytest.raises(RequestsRespectfulConfigError): RespectfulRequester.configure(requests_module_name=100) RespectfulRequester.configure(requests_module_name="requests") RespectfulRequester.configure_default()
def __init__(self): self.rr = RespectfulRequester() self.rr.register_realm("sochain", max_requests=299, timespan=60) self.requestCount = 0
def test_the_class_should_be_able_to_restore_the_default_configuration_values(): rr = RespectfulRequester() RespectfulRequester.configure(safety_threshold=20) RespectfulRequester.configure(requests_module_name="r") RespectfulRequester.configure(redis={"host": "0.0.0.0", "port": 6379, "database": 1}) RespectfulRequester.configure_default() assert rr._config()["safety_threshold"] == 10 assert rr._config()["requests_module_name"] == "requests" assert rr._config()["redis"]["host"] == "localhost" assert rr._config()["redis"]["database"] == 0
OH_API_BASE = OPENHUMANS_OH_BASE_URL + '/api/direct-sharing' OH_DIRECT_UPLOAD = OH_API_BASE + '/project/files/upload/direct/' OH_DIRECT_UPLOAD_COMPLETE = OH_API_BASE + '/project/files/upload/complete/' OH_DELETE_FILES = OH_API_BASE + '/project/files/delete/' # Requests Respectful (rate limiting, waiting) if REMOTE is True: from urllib.parse import urlparse url_object = urlparse(os.getenv('REDIS_URL')) logger.info('Connecting to redis at %s:%s', url_object.hostname, url_object.port) RespectfulRequester.configure(redis={ "host": url_object.hostname, "port": url_object.port, "password": url_object.password, "database": 0 }, safety_threshold=5) # This creates a Realm called "source" that allows 60 requests per minute maximum. rr = RespectfulRequester() rr.register_realm("Source", max_requests=60, timespan=60) # Applications installed INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages',