def test_get_documents_depaginate(self):
docs = (FakeRestResponse(
content={
'data': ['this', 'that'],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=2'
}
}
}),
FakeRestResponse(
content={
'data': ['other', 'another'],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=3'
}
}
}),
FakeRestResponse(content={
'data': ['more', 'things'],
'_links': {}
}))
patched_req = patch(ppath('_req'), side_effect=docs)
with patched_req as mocked_req:
assert self.comm.get_documents('an_endpoint',
all_pages=True,
max_results=101) == [
'this', 'that', 'other',
'another', 'more', 'things'
]
assert all([
a[0][1].startswith(rest_url('an_endpoint'))
for a in mocked_req.call_args_list
])
assert [a[1] for a in mocked_req.call_args_list] == [
# Communicator.get_content passes ints
{
'params': {
'page': 1,
'max_results': 101
},
'quiet': False
},
# url parsing passes strings, but requests removes the quotes anyway
{
'params': {
'page': '2',
'max_results': '101'
},
'quiet': False
},
{
'params': {
'page': '3',
'max_results': '101'
},
'quiet': False
}
]
def fake_request(method, url, **kwargs):
if kwargs.get('files'):
if 'json' in kwargs:
raise Exception
if 'data' in kwargs and check_if_nested(kwargs['data']):
raise Exception
return FakeRestResponse(test_nested_request_content)
def test_communication_error(self, mocked_req, mocked_log):
response = FakeRestResponse({})
response.status_code = 500
mocked_req.return_value = response
self.comm.lock = MagicMock()
self.comm.lock.__enter__.assert_not_called()
self.comm.lock.__exit__.assert_not_called()
with self.assertRaises(RestCommunicationError) as e:
self.comm.get_document('an_endpoint')
assert mocked_log.call_args[0][0].endswith(
'Status code 500. Reason: a reason')
assert str(e.exception) == 'Encountered a 500 status code: a reason'
self.comm.lock.__enter__.assert_called_once()
self.comm.lock.__exit__.assert_called_once(
) # exception raised, but lock still released
def test_fetch_from_eutils():
ncbi_search_data = {'esearchresult': {'idlist': ['1337']}}
ncbi_fetch_data = '''
<ScientificName>Genus species</ScientificName>
<OtherNames><CommonName>a common name</CommonName></OtherNames>
<Rank>species</Rank>
'''
ncbi_fetch_data_sub_spe = '''
<ScientificName>Genus species</ScientificName>
<OtherNames><CommonName>a common name</CommonName></OtherNames>
<Rank>subspecies</Rank>
'''
patched_get = patch(
'egcg_core.ncbi.requests.get',
side_effect=(FakeRestResponse(content=ncbi_search_data),
FakeRestResponse(content=ncbi_fetch_data),
FakeRestResponse(content=ncbi_fetch_data),
FakeRestResponse(content=ncbi_fetch_data)))
patched_get2 = patch(
'egcg_core.ncbi.requests.get',
side_effect=(FakeRestResponse(content=ncbi_search_data),
FakeRestResponse(content=ncbi_fetch_data_sub_spe),
FakeRestResponse(content=ncbi_fetch_data_sub_spe),
FakeRestResponse(content=ncbi_fetch_data_sub_spe)))
with patched_get as mocked_get:
obs = fetch_from_eutils('a_species')
assert obs == ('1337', 'Genus species', 'a common name')
mocked_get.assert_any_call(
'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi',
params={
'db': 'Taxonomy',
'term': 'a_species',
'retmode': 'JSON'
})
mocked_get.assert_any_call(
'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi',
params={
'db': 'Taxonomy',
'id': '1337'
})
with patched_get2:
obs = fetch_from_eutils('a_species')
assert obs == ('1337', 'Genus species', 'a common name')
def ppath(extension):
return 'egcg_core.rest_communication.Communicator.' + extension
test_endpoint = 'an_endpoint'
test_request_content = {'data': ['some', {'test': 'content'}]}
test_patch_document = {
'_id': '1337',
'_etag': 1234567,
'uid': 'a_unique_id',
'list_to_update': ['this', 'that', 'other']
}
patched_response = patch('requests.request',
return_value=FakeRestResponse(
status_code=200, content=test_request_content))
auth = ('a_user', 'a_password')
class TestRestCommunication(TestEGCG):
def setUp(self):
self.comm = rest_communication.Communicator(
auth=auth, baseurl='http://localhost:4999/api/0.1')
def test_api_url(self):
assert self.comm.api_url('an_endpoint') == rest_url('an_endpoint')
def test_parse_query_string(self):
query_string = 'http://a_url?this=that&other={"another":"more"}&things=1'
no_query_string = 'http://a_url'
dodgy_query_string = 'http://a_url?this=that?other=another'
class TestRestCommunication(TestEGCG):
def setUp(self):
self.comm = rest_communication.Communicator(
auth=auth, baseurl='http://localhost:4999/api/0.1')
def test_begin_session(self):
s = self.comm.begin_session()
assert s.adapters['http://'] is s.adapters['https://']
assert s.auth == auth
assert s.params == {}
hashed_token = '{"some": "hashed"}.tokenauthentication'
self.comm._auth = hashed_token
s = self.comm.begin_session()
assert s.headers['Authorization'] == 'Token ' + hashed_token
def test_api_url(self):
assert self.comm.api_url('an_endpoint') == test_url
def test_parse_query_string(self):
query_string = 'http://a_url?this=that&other={"another":"more"}&things=1'
dodgy_query_string = 'http://a_url?this=that?other=another'
assert self.comm._parse_query_string('http://a_url') == {}
assert self.comm._parse_query_string(query_string) == {
'this': 'that',
'other': '{"another":"more"}',
'things': '1'
}
with self.assertRaises(RestCommunicationError) as e:
self.comm._parse_query_string(dodgy_query_string)
assert str(
e.exception) == 'Bad query string: ' + dodgy_query_string
with self.assertRaises(RestCommunicationError) as e2:
self.comm._parse_query_string(query_string, requires=['thangs'])
assert str(
e2.exception
) == query_string + " did not contain all required fields: ['thangs']"
def test_detect_files_in_json(self):
json_no_files = {'k1': 'v1', 'k2': 'v2'}
obs_files, obs_json = self.comm._detect_files_in_json(json_no_files)
assert obs_files is None
assert obs_json == json_no_files
file_path = os.path.join(self.assets_path, 'test_to_upload.txt')
json_with_files = {'k1': 'v1', 'k2': ('file', file_path)}
obs_files, obs_json = self.comm._detect_files_in_json(json_with_files)
assert obs_files == {'k2': (file_path, b'test content', 'text/plain')}
assert obs_json == {'k1': 'v1'}
json_list = [json_with_files, json_with_files]
obs_files, obs_json = self.comm._detect_files_in_json(json_list)
assert obs_files == [{
'k2': (file_path, b'test content', 'text/plain')
}, {
'k2': (file_path, b'test content', 'text/plain')
}]
assert obs_json == [{'k1': 'v1'}, {'k1': 'v1'}]
@patched_request
def test_req(self, mocked_request):
json_content = ['some', {'test': 'json'}]
response = self.comm._req('METHOD', test_url, json=json_content)
assert response.status_code == 200
assert json.loads(response.content.decode(
'utf-8')) == response.json() == test_nested_request_content
mocked_request.assert_called_with('METHOD',
test_url,
json=json_content)
@patch('egcg_core.rest_communication.sleep')
@patch.object(Session,
'request',
side_effect=[
SSLError('Error 1'),
SSLError('Error 2'),
FakeRestResponse(test_nested_request_content)
])
def test_retry(self, mocked_request, mocked_sleep):
assert self.comm._req('METHOD', test_url,
json=[]).json() == test_nested_request_content
c = call('METHOD', test_url, json=[])
mocked_request.assert_has_calls([c, c, c])
assert mocked_sleep.call_count == 2
@patch('egcg_core.rest_communication.sleep')
@patched_failed_request
def test_failed_req(self, mocked_request, mocked_sleep):
json_content = ['some', {'test': 'json'}]
self.comm.lock = MagicMock()
self.comm.lock.__enter__.assert_not_called()
self.comm.lock.__exit__.assert_not_called()
with pytest.raises(SSLError):
_ = self.comm._req('METHOD', test_url, json=json_content)
assert self.comm.lock.__enter__.call_count == 6
assert self.comm.lock.__exit__.call_count == 6 # exception raised, but lock still released
mocked_sleep.assert_called_with(2)
assert mocked_sleep.call_count == 5
@patched_request
def test_multi_session(self, mocked_request):
json_content = ['some', {'test': 'json'}]
with patch('os.getpid', return_value=1):
_ = self.comm._req('METHOD', test_url, json=json_content)
with patch('os.getpid', return_value=2):
_ = self.comm._req('METHOD', test_url, json=json_content)
assert len(self.comm._sessions) == 2
@patched_request
def test_with_multiprocessing(self, mocked_request):
json_content = ['some', {'test': 'json'}]
def assert_request():
_ = self.comm._req('METHOD', test_url, json=json_content)
assert mocked_request.call_count == 2
assert len(self.comm._sessions) == 2
# initiate in the Session in the main thread
self.comm._req('METHOD', test_url, json=json_content)
procs = []
for i in range(10):
procs.append(multiprocessing.Process(target=assert_request))
for p in procs:
p.start()
for p in procs:
p.join()
@patch.object(Session, '__exit__')
@patch.object(Session, '__enter__')
@patched_request
def test_context_manager(self, mocked_request, mocked_enter, mocked_exit):
json_content = ['some', {'test': 'json'}]
with self.comm.session:
mocked_enter.assert_called_once()
mocked_exit.assert_not_called()
for i in range(4): # multiple calls
response = self.comm._req('METHOD',
test_url,
json=json_content)
assert response.status_code == 200
assert response.json() == test_nested_request_content
mocked_request.assert_called_with('METHOD',
test_url,
json=json_content)
assert mocked_request.call_count == 4
mocked_exit.assert_called_once()
@patch.object(rest_communication.Communicator, 'error')
@patch.object(Session, 'request')
def test_communication_error(self, mocked_req, mocked_log):
response = FakeRestResponse({})
response.status_code = 500
mocked_req.return_value = response
self.comm.lock = MagicMock()
self.comm.lock.__enter__.assert_not_called()
self.comm.lock.__exit__.assert_not_called()
with self.assertRaises(RestCommunicationError) as e:
self.comm.get_document('an_endpoint')
assert mocked_log.call_args[0][0].endswith(
'Status code 500. Reason: a reason')
assert str(e.exception) == 'Encountered a 500 status code: a reason'
self.comm.lock.__enter__.assert_called_once()
self.comm.lock.__exit__.assert_called_once(
) # exception raised, but lock still released
@patch.object(rest_communication.Communicator, '_req')
def test_get_documents_depaginate(self, mocked_req):
mocked_req.side_effect = (
FakeRestResponse({
'data': ['this', 'that'],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=2'
}
}
}),
FakeRestResponse({
'data': ['other', 'another'],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=3'
}
}
}), FakeRestResponse({
'data': ['more', 'things'],
'_links': {}
}))
assert self.comm.get_documents('an_endpoint',
all_pages=True,
max_results=101) == [
'this', 'that', 'other', 'another',
'more', 'things'
]
mocked_req.assert_has_calls((
# Communicator.get_content passes ints
call('GET',
test_url,
params={
'page': 1,
'max_results': 101
},
quiet=False),
# url parsing passes strings, but requests removes the quotes anyway
call('GET',
test_url,
params={
'page': '2',
'max_results': '101'
},
quiet=False),
call('GET',
test_url,
params={
'page': '3',
'max_results': '101'
},
quiet=False)))
docs = [
FakeRestResponse({
'data': ['data%s' % d],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=%s' % d
}
}
}) for d in range(1, 1200)
]
docs.append(FakeRestResponse({'data': ['last piece'], '_links': {}}))
mocked_req.side_effect = docs
ret = self.comm.get_documents('an_endpoint',
all_pages=True,
max_results=101)
assert len(ret) == 1200
@patched_request
def test_get_content(self, mocked_request):
data = self.comm.get_content(test_endpoint,
max_results=100,
where={'a_field': 'thing'})
assert data == test_nested_request_content
mocked_request.assert_called_with('GET',
test_url,
params={
'max_results': 100,
'where': '{"a_field": "thing"}',
'page': 1
})
def test_get_documents(self):
with patched_request:
data = self.comm.get_documents(test_endpoint,
max_results=100,
where={'a_field': 'thing'})
assert data == test_nested_request_content['data']
def test_get_document(self):
expected = test_nested_request_content['data'][0]
with patched_request:
observed = self.comm.get_document(test_endpoint,
max_results=100,
where={'a_field': 'thing'})
assert observed == expected
@patched_request
def test_post_entry(self, mocked_request):
self.comm.post_entry(test_endpoint,
payload=test_nested_request_content)
mocked_request.assert_called_with('POST',
test_url,
json=test_nested_request_content,
files=None)
file_path = os.path.join(self.assets_path, 'test_to_upload.txt')
test_request_content_plus_files = dict(test_flat_request_content)
test_request_content_plus_files['f'] = ('file', file_path)
self.comm.post_entry(test_endpoint,
payload=test_request_content_plus_files)
mocked_request.assert_called_with(
'POST',
test_url,
data=test_flat_request_content,
files={'f': (file_path, b'test content', 'text/plain')})
self.comm.post_entry(test_endpoint,
payload=test_flat_request_content,
use_data=True)
mocked_request.assert_called_with('POST',
test_url,
data=test_flat_request_content,
files=None)
self.comm.post_entry(test_endpoint,
payload=test_request_content_plus_files,
use_data=True)
mocked_request.assert_called_with(
'POST',
test_url,
data=test_flat_request_content,
files={'f': (file_path, b'test content', 'text/plain')})
@patched_request
def test_put_entry(self, mocked_request):
self.comm.put_entry(test_endpoint,
'an_element_id',
payload=test_nested_request_content)
mocked_request.assert_called_with('PUT',
test_url + 'an_element_id',
json=test_nested_request_content,
files=None)
file_path = os.path.join(self.assets_path, 'test_to_upload.txt')
test_request_content_plus_files = dict(test_flat_request_content)
test_request_content_plus_files['f'] = ('file', file_path)
self.comm.put_entry(test_endpoint,
'an_element_id',
payload=test_request_content_plus_files)
mocked_request.assert_called_with(
'PUT',
test_url + 'an_element_id',
data=test_flat_request_content,
files={'f': (file_path, b'test content', 'text/plain')})
@patch.object(rest_communication.Communicator,
'get_document',
return_value=test_patch_document)
@patched_request
def test_patch_entry(self, mocked_request, mocked_get_doc):
patching_payload = {'list_to_update': ['another']}
self.comm.patch_entry(test_endpoint,
payload=patching_payload,
id_field='uid',
element_id='a_unique_id',
update_lists=['list_to_update'])
mocked_get_doc.assert_called_with(test_endpoint,
where={'uid': 'a_unique_id'})
mocked_request.assert_called_with(
'PATCH',
test_url + '1337',
headers={'If-Match': 1234567},
json={'list_to_update': ['this', 'that', 'other', 'another']},
files=None)
@patch.object(rest_communication.Communicator,
'get_document',
return_value=test_patch_document)
@patched_request
def test_if_match(self, mocked_request, mocked_get_doc):
self.comm.patch_entry(test_endpoint, {'this': 'that'}, 'uid',
'a_unique_id')
mocked_get_doc.assert_called_with(test_endpoint,
where={'uid': 'a_unique_id'})
mocked_request.assert_called_with('PATCH',
test_url + '1337',
headers={'If-Match': 1234567},
json={'this': 'that'},
files=None)
@patch.object(rest_communication.Communicator,
'post_entry',
return_value=True)
@patch.object(rest_communication.Communicator,
'_patch_entry',
return_value=True)
@patch.object(rest_communication.Communicator, 'get_document')
def test_post_or_patch(self, mocked_get, mocked_patch, mocked_post):
test_post_or_patch_payload = {
'uid': '1337',
'list_to_update': ['more'],
'another_field': 'that'
}
test_post_or_patch_payload_no_uid = {
'list_to_update': ['more'],
'another_field': 'that'
}
test_post_or_patch_doc = {
'uid': 'a_uid',
'_id': '1337',
'_etag': 1234567,
'list_to_update': ['things'],
'another_field': 'this'
}
mocked_get.return_value = test_post_or_patch_doc
self.comm.post_or_patch('an_endpoint', [test_post_or_patch_payload],
id_field='uid',
update_lists=['list_to_update'])
mocked_get.assert_called_with('an_endpoint', where={'uid': '1337'})
mocked_patch.assert_called_with('an_endpoint', test_post_or_patch_doc,
test_post_or_patch_payload_no_uid,
['list_to_update'])
mocked_get.return_value = None
self.comm.post_or_patch('an_endpoint', [test_post_or_patch_payload],
id_field='uid',
update_lists=['list_to_update'])
mocked_get.assert_called_with('an_endpoint', where={'uid': '1337'})
mocked_post.assert_called_with('an_endpoint',
test_post_or_patch_payload)
def test_get_documents_depaginate(self, mocked_req):
mocked_req.side_effect = (
FakeRestResponse({
'data': ['this', 'that'],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=2'
}
}
}),
FakeRestResponse({
'data': ['other', 'another'],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=3'
}
}
}), FakeRestResponse({
'data': ['more', 'things'],
'_links': {}
}))
assert self.comm.get_documents('an_endpoint',
all_pages=True,
max_results=101) == [
'this', 'that', 'other', 'another',
'more', 'things'
]
mocked_req.assert_has_calls((
# Communicator.get_content passes ints
call('GET',
test_url,
params={
'page': 1,
'max_results': 101
},
quiet=False),
# url parsing passes strings, but requests removes the quotes anyway
call('GET',
test_url,
params={
'page': '2',
'max_results': '101'
},
quiet=False),
call('GET',
test_url,
params={
'page': '3',
'max_results': '101'
},
quiet=False)))
docs = [
FakeRestResponse({
'data': ['data%s' % d],
'_links': {
'next': {
'href': 'an_endpoint?max_results=101&page=%s' % d
}
}
}) for d in range(1, 1200)
]
docs.append(FakeRestResponse({'data': ['last piece'], '_links': {}}))
mocked_req.side_effect = docs
ret = self.comm.get_documents('an_endpoint',
all_pages=True,
max_results=101)
assert len(ret) == 1200