def test_get_handles_exception(monkeypatch): def mock_requests_get(url, params, **kwargs): raise requests.exceptions.ReadTimeout('test timeout!') monkeypatch.setattr(requester.requests, 'get', mock_requests_get) dq = requester.DelayedRequester(1) dq.get('https://google.com/')
def test_get_waits_before_getting(monkeypatch): delay = 0.2 def mock_requests_get(url, params, **kwargs): return requests.Response() monkeypatch.setattr(requester.requests, 'get', mock_requests_get) dq = requester.DelayedRequester(delay) s = time.time() dq.get('https://google.com') print(time.time() - s) start = time.time() dq.get('https://google.com') assert time.time() - start >= delay
def test_get_response_json_retries_with_none_response(): dq = requester.DelayedRequester(1) with patch.object( dq, 'get', return_value=None ) as mock_get: with pytest.raises(Exception): assert dq.get_response_json( 'https://google.com/', retries=2, ) assert mock_get.call_count == 3
def test_get_response_json_retries_with_error_json(): dq = requester.DelayedRequester(1) r = requests.Response() r.status_code = 200 r.json = MagicMock(return_value={'error': ''}) with patch.object( dq, 'get', return_value=r ) as mock_get: with pytest.raises(Exception): assert dq.get_response_json( 'https://google.com/', retries=2, ) assert mock_get.call_count == 3
def test_get_response_json_returns_response_json_when_all_ok(): dq = requester.DelayedRequester(1) expect_response_json = {'batchcomplete': ''} r = requests.Response() r.status_code = 200 r.json = MagicMock(return_value=expect_response_json) with patch.object( dq, 'get', return_value=r ) as mock_get: actual_response_json = dq.get_response_json( 'https://google.com/', retries=2, ) assert mock_get.call_count == 1 assert actual_response_json == expect_response_json
import argparse import common.requester as requester import common.storage.image as image import logging DELAY = 1.0 # time delay (in seconds) PROVIDER = 'met' ENDPOINT = 'https://collectionapi.metmuseum.org/public/collection/v1/objects' logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s: %(message)s', level=logging.INFO) logger = logging.getLogger(__name__) delayed_requester = requester.DelayedRequester(DELAY) image_store = image.ImageStore(provider=PROVIDER) def main(date=None): """ This script pulls the data for a given date from the Metropolitan Museum of Art API, and writes it into a .TSV file to be eventually read into our DB. Required Arguments: date: Date String in the form YYYY-MM-DD. This is the date for which running the script will pull data. """