def test_record_or_load_remakes(self, mock_requests, monkeypatch):
mock_requests.request.return_value = mock.Mock(
encoding='utf-8',
content='rawr',
status_code=200,
headers={'tota': 'dyle'})
requests.HarvesterResponse(ok=False,
method='get',
url='dinosaurs.sexy').save()
model = requests.HarvesterResponse.get(method='get',
url='dinosaurs.sexy')
assert not model.ok
assert model.method == 'get'
assert model.url == 'dinosaurs.sexy'
resp = requests.get('dinosaurs.sexy')
model = requests.HarvesterResponse.get(method='get',
url='dinosaurs.sexy')
assert model.method == 'get'
assert model.content == b'rawr'
assert model.encoding == 'utf-8'
assert model.status_code == 200
assert model.url == 'dinosaurs.sexy'
assert model.headers == {'tota': 'dyle'}
assert model.headers_str == '{"tota": "dyle"}'
assert isinstance(resp, requests.HarvesterResponse)
def test_text_is_unicode(self):
resp = requests.HarvesterResponse(method='POST',
url='dinosaurs.sexy',
content='probably xml')
assert isinstance(resp.text, six.text_type)
assert resp.text == u'probably xml'
def test_json_works(self):
data = {'totally': 'dyle'}
resp = requests.HarvesterResponse(content=json.dumps(data))
assert resp.json() == data
assert resp.content == json.dumps(data)
def test_record_or_load_loads(self, mock_requests, monkeypatch):
requests.HarvesterResponse(ok=True, method='get', url='dinosaurs.sexy', content='rawr', headers_str="{}").save()
resp = requests.get('dinosaurs.sexy')
assert resp.headers == {}
assert resp.content == 'rawr'
assert not mock_requests.request.called
assert isinstance(resp, requests.HarvesterResponse)
def test_json_works(self):
data = {'totally': 'dyle'}
resp = requests.HarvesterResponse(method='POST',
url='dinosaurs.sexy',
content=json.dumps(data))
assert resp.json() == data
assert resp.content == json.dumps(data)
def test_request_doesnt_throttle_on_load(self, mock_requests, monkeypatch):
mock_sleep = mock.Mock()
monkeypatch.setattr(requests.time, 'sleep', mock_sleep)
requests.HarvesterResponse(ok=True, method='get', url='dinosaurs.sexy', content='citychicken').save()
resp = requests.get('dinosaurs.sexy', throttle=2)
assert mock_sleep.called is False
assert mock_requests.request.called is False
assert isinstance(resp, requests.HarvesterResponse)
def mock_maybe_load_response(monkeypatch):
mock_mlr = mock.Mock()
mock_mlr.return_value = requests.HarvesterResponse(
ok=True,
method='get',
url=request_url.lower(),
content=TEST_OAI_DOC,
)
mock_save = lambda x: x
monkeypatch.setattr(requests, '_maybe_load_response', mock_mlr)
monkeypatch.setattr(requests.HarvesterResponse, 'save', mock_save)
def test_force_makes_new_request(self, mock_requests, monkeypatch):
requests.HarvesterResponse(ok=True, method='get', url='dinosaurs.sexy', content='citychicken').save()
mock_requests.request.return_value = mock.Mock(encoding='utf-8', content='Snapcity', status_code=200, headers={'tota': 'dyle'})
resp = requests.get('dinosaurs.sexy')
assert resp.content == 'citychicken'
assert mock_requests.request.called is False
resp = requests.get('dinosaurs.sexy', force=True)
assert resp.content == 'Snapcity'
assert mock_requests.request.called is True
def harvest(self, mock_requests, start_date=None, end_date=None):
request_url = 'http://validOAI.edu/?sonofaplumber'
requests.HarvesterResponse(
ok=True,
method='get',
url=request_url.lower(),
content=TEST_OAI_DOC,
content_type="application/XML"
).save()
start_date = date(2015, 3, 14)
end_date = date(2015, 3, 16)
records = self.get_records(request_url, start_date, end_date)
return [RawDocument({
'doc': TEST_OAI_DOC,
'source': 'test',
'filetype': 'XML',
'docID': "1"
}) for record in records]
def test_text_is_unicode(self):
resp = requests.HarvesterResponse(content='probably xml')
assert isinstance(resp.text, six.text_type)
assert resp.text == u'probably xml'
def test_text_works(self):
resp = requests.HarvesterResponse(content='probably xml')
assert resp.text == 'probably xml'
def test_text_is_unicode(self):
resp = requests.HarvesterResponse(content='probably xml')
assert isinstance(resp.text, unicode)
assert resp.text == 'probably xml'.decode('utf-8')
def test_text_works(self):
resp = requests.HarvesterResponse(method='POST',
url='dinosaurs.sexy',
content='probably xml')
assert resp.text == 'probably xml'
