def test_existing_gf_id(event, obj):
""" Test that a new genomic file is created with predefined kf_id """
os.environ['DATASERVICE_API'] = 'http://api.com/'
obj()
s3 = boto3.client('s3')
# Add a gf_id
tags = TAGS.copy()
tags['TagSet'].append({'Key': 'gf_id', 'Value': 'GF_00000002'})
response = s3.put_object_tagging(Bucket=BUCKET, Key=OBJECT, Tagging=tags)
mock = patch('service.requests')
req = mock.start()
def mock_get(url, *args, **kwargs):
if '/genomic-files' in url:
resp = MagicMock()
resp.status_code = 404
return resp
elif '/biospecimens' in url:
resp = MagicMock()
resp.json.return_value = {'results': {'kf_id': url[:-11]}}
resp.status_code = 200
return resp
elif '/studies' in url:
resp = MagicMock()
resp.status_code = 200
return resp
req.get.side_effect = mock_get
mock_resp = MagicMock()
mock_resp.json.return_value = {'results': {'kf_id': 'GF_00000000'}}
mock_resp.status_code = 201
req.post.return_value = mock_resp
service.handler(event, {})
expected = {
'kf_id': 'GF_00000002',
'file_name': '60d33dec-98db-446c-ac64-f4d027588f26.cram',
'file_format': 'cram',
'acl': ['SD_9PYZAHHE'],
'data_type': 'Aligned Reads',
'controlled_access': True,
'availability': 'Immediate Download',
'is_harmonized': True,
'reference_genome': 'GRCh38',
'biospecimen_id': 'BS_QV3Z0DZM',
'hashes': {
'etag': 'd41d8cd98f00b204e9800998ecf8427e'
},
'size': 1024,
'urls': ['s3://{}/{}'.format(BUCKET, OBJECT)]
}
assert req.get.call_count == 4
req.post.assert_any_call('http://api.com/genomic-files', json=expected)
assert req.post.call_count == 2
mock.stop()
def test_api_key_set(monkeypatch):
""" Ensures exception raised if api key is not set
"""
monkeypatch.delenv("API_KEY", raising=False)
event = dict(freshen_url="doesn't matter")
with pytest.raises(KeyError):
handler(event, None)
def test_records_none(self):
testRec = {'source': 'Kinesis'}
try:
handler(testRec, None)
except NoRecordsReceived:
pass
self.assertRaises(NoRecordsReceived)
def test_out_of_time(event, obj):
""" Test that a function is re-invoked when records remain """
obj()
os.environ['DATASERVICE_API'] = 'http://api.com/'
mock_r = patch('service.requests')
req = mock_r.start()
class Context:
def __init__(self):
self.invoked_function_arn = 'arn:aws:lambda:::function:kf-lambda'
def get_remaining_time_in_millis(self):
return 300
# Add a second record
event['Records'].append(event['Records'][0])
with patch('service.boto3.client') as mock:
service.handler(event, Context())
assert mock().invoke.call_count == 1
_, args = mock().invoke.call_args_list[0]
assert args['FunctionName'] == Context().invoked_function_arn
assert args['InvocationType'] == 'Event'
payload = json.loads(args['Payload'].decode('utf-8'))
assert payload == {'Records': [event['Records'][1]]}
mock_r.stop()
def test_handler_error(self):
testRec = {'source': 'Kinesis', 'Records': []}
try:
handler(testRec, None)
except NoRecordsReceived:
pass
self.assertRaises(NoRecordsReceived)
def test_upload(obj, messages, req, env):
"""
Test that compressed log file is created
"""
s3 = obj()
service.handler({}, Context(40000))
assert len(s3.list_objects(Bucket=BUCKET)['Contents']) == 1
stream = io.BytesIO()
KEY = s3.list_objects(Bucket=BUCKET)['Contents'][0]['Key']
ob = s3.get_object(Bucket=BUCKET, Key=KEY)
#dat = gzip.decompress(ob['Body'].read()).decode()
dat = ob['Body'].read().decode()
for i, l in enumerate(dat.split('\n')):
assert json.loads(messages[i])['Message'].replace("'", '"') == l
message = json.loads(l)
assert 'path' in message
assert 'method' in message
assert 'api_version' in message
assert 'api_commit' in message
assert 'data' in message
assert req.post.call_count == 2
def test_recent_adoptions(self):
data = {
"username": "******",
"password": "******",
"method": "xml_recent_adoptions"
}
post = utils.PostedData(data, "en")
service.handler(post, "127.0.0.1", "sheltermanager.com")
data["method"] = "json_recent_adoptions"
service.handler(post, "127.0.0.1", "sheltermanager.com")
def test_handler_error(self):
testRec = {
'httpMethod': 'GET',
'queryStringParameters': {
'nonISBN': '987.23'
}
}
try:
handler(testRec, None)
except InvalidExecutionType:
pass
self.assertRaises(InvalidExecutionType)
def test_handler_error(self):
testRec = {
'httpMethod': 'GET',
'queryStringParameters': {
'otherQuery': 'Tester, Test'
}
}
try:
handler(testRec, None)
except InvalidExecutionType:
pass
self.assertRaises(InvalidExecutionType)
def test_delete(obj, messages, queue, req, env):
"""
Test that messages are deleted after being written
"""
s3 = obj()
queue_url = queue()
service.handler({}, Context(40000))
sqs = boto3.resource('sqs', region_name='us-east-1')
queue = sqs.Queue(queue_url)
batch = queue.receive_messages(MaxNumberOfMessages=10,
WaitTimeSeconds=0)
assert len(batch) == 0
assert req.post.call_count == 2
def test_query_filter_mapping(monkeypatch):
monkeypatch.setenv("BITLY_GENERIC_ACCESS_TOKEN", "my_token")
url = dict(url="https://arxiv.org/")
event = dict(queryStringParameters=url)
result_dict = handler(event, None)
expected_dict = {'statusCode': 200, 'body': '{"id": "https://j.mp/2soOxSO"}'}
assert result_dict == expected_dict
def test_handler_empty(self, mock_fetch):
testRec = {
'source': 'Kinesis',
'Records': [{'some': 'record'}]
}
resp = handler(testRec, None)
mock_fetch.assert_called_once()
self.assertEqual(resp[0][0], 'empty')
def test_handler_handles_http_error(mocker, event):
"""Ensures any HTTP client errors get handled"""
mocker.patch(
"service.urlopen",
side_effect=HTTPError("http://example.com", 500, "Internal Error", {},
None),
)
assert handler(event, None) is False
def test_handler_clean(self, mocker):
with patch.multiple(SourceManager,
fetchRecords=DEFAULT,
sendWorksToKinesis=DEFAULT) as managerMocks:
outWorks = handler({}, {})
managerMocks['fetchRecords'].assert_called_once()
managerMocks['sendWorksToKinesis'].assert_called_once()
assert outWorks == []
def test_handler_raises(monkeypatch):
"""Reads from the `test_handler_raises` cassette and processes the request that raises
"""
monkeypatch.setenv("API_KEY", "mykey")
event = dict(
freshen_url=
"https://hostdoesnotexist.gov/v1/some_endpoint/freshen.json?api_key=")
resp = handler(event, None)
assert resp is False
def test_reinvoke(obj, messages, queue, req, env):
"""
Test that messages are deleted after being written
"""
s3 = obj()
queue_url = queue()
patch_lam = patch('boto3.client')
mock_lam = patch_lam.start()
service.handler({}, Context(2000))
sqs = boto3.resource('sqs', region_name='us-east-1')
queue = sqs.Queue(queue_url)
assert mock_lam().invoke.call_count == 1
assert mock_lam().put_object.call_count == 1
assert req.post.call_count == 2
mock_lam.stop()
def test_handler(monkeypatch):
"""Reads from the `test_handler` cassette and processes the request
"""
monkeypatch.setenv("API_KEY", "mykey")
event = dict(
freshen_url=
"https://api.trade.gov/v1/some_endpoint/freshen.json?api_key=")
resp = handler(event, None)
assert resp is True
def test_handler_local(self, mock_parser, mock_load):
testRec = {
'source': 'local.file',
'localFile': 'test.csv'
}
resp = handler(testRec, None)
mock_load.assert_called_once()
mock_parser.assert_called_once()
self.assertTrue(resp)
def test_handler():
"""
Test the service handler
"""
r_patch = patch('service.requests')
r_mock = r_patch.start()
mock_resp = Mock()
mock_resp.status_code = 200
r_mock.get.return_value = mock_resp
event = {
'service': 'http://kids-first.github.io',
'endpoint': '/'
}
service.handler(event, {})
assert r_mock.get.call_count == 1
r_mock.get.assert_called_with('http://kids-first.github.io/',
headers={'User-Agent': 'HealthChecker'})
def test_handler_scheduled(self, mock_parser, mock_fetch):
testRec = {
'source': 'Kinesis',
'Records': [{'some': 'record'}]
}
resp = handler(testRec, None)
mock_fetch.assert_called_once()
mock_parser.assert_called_once()
self.assertTrue(resp)
def test_handler_clean(self, mockResponse, mockFetch):
testRec = {
'queryStringParameters': {
'identifier': '000000000',
'type': 'oclc'
}
}
resp = handler(testRec, None)
self.assertEqual(resp, 'outObject')
mockFetch.assert_called_once_with('000000000', 'oclc')
mockResponse.assert_called_once_with(200, 'oclcResponse')
def test_handler_clean(self):
testRec = {
'source': 'Kinesis',
'Records': [{
'kinesis': {
'data': 'data'
}
}]
}
resp = handler(testRec, None)
self.assertEqual(resp, 'Hello, World')
def test_handler_internal_error(self, mockResponse, mockFetch):
testRec = {
'queryStringParameters': {
'identifier': '000000000',
'type': 'oclc'
}
}
resp = handler(testRec, None)
self.assertEqual(resp, 'outObject')
mockFetch.assert_called_once_with('000000000', 'oclc')
mockResponse.assert_called_once_with(500, {'message': 'Test Error'})
def test_handler_clean(self, mock_parse):
testRec = {
'source': 'Kinesis',
'Records': [{
'kinesis': {
'data': 'data'
}
}]
}
resp = handler(testRec, None)
mock_parse.assert_called_once()
def test_handler_local(self, mock_parser, mock_load):
testRec = {
'source': 'local.file',
'localFile': 'test.csv',
'start': 0,
'size': 1
}
resp = handler(testRec, None)
mock_load.assert_called_once()
mock_parser.assert_called_once()
self.assertEqual(resp, [1, 2])
def test_handler_error_bad_parameters(self, mockResponse):
testRec = {
'queryStringParameters': {
'identifier': '000000000',
}
}
resp = handler(testRec, None)
self.assertEqual(resp, 'outObject')
mockResponse.assert_called_once_with(
400,
{'message': 'GET query must include identifier and type'}
)
def test_viaf_query_error(self, mock_viaf):
testRec = {
'httpMethod': 'GET',
'queryStringParameters': {
'queryName': 'Tester, Test'
}
}
mock_viaf().query.side_effect = VIAFError('test viaf error')
mock_viaf.formatResponse.return_value = {
'status': 500,
'data': 'test error'
}
resp = handler(testRec, None)
self.assertEqual(resp['status'], 500)
def test_existing_gf(event, obj):
""" Test that a new genomic file is not created if it exists """
obj()
s3 = boto3.client('s3')
# Add a gf_id
tags = TAGS.copy()
tags['TagSet'].append({'Key': 'gf_id', 'Value': 'GF_00000001'})
response = s3.put_object_tagging(Bucket=BUCKET, Key=OBJECT, Tagging=tags)
os.environ['DATASERVICE_API'] = 'http://api.com/'
mock = patch('service.requests')
req = mock.start()
mock_resp = MagicMock()
mock_resp.json.return_value = {'results': {'kf_id': 'GF_00000001'}}
mock_resp.status_code = 200
req.get.return_value = mock_resp
service.handler(event, {})
assert req.get.call_count == 1
assert req.post.call_count == 0
mock.stop()
def test_unglue_query_error(self, mock_unglue):
testRec = {
'httpMethod': 'GET',
'queryStringParameters': {
'isbn': '9999999999'
}
}
mock_unglue().fetchSummary.side_effect = UnglueError(
500, 'test unglue error')
mock_unglue.formatResponse.return_value = {
'status': 500,
'data': 'test error'
}
resp = handler(testRec, None)
self.assertEqual(resp['status'], 500)
def test_handler_clean(self, mock_viaf):
testRec = {
'httpMethod': 'GET',
'queryStringParameters': {
'queryName': 'Tester, Test'
}
}
mock_viaf().query.return_value = {
'status': 200,
'data': {
'test': 'test'
}
}
resp = handler(testRec, None)
self.assertEqual(resp['status'], 200)
self.assertEqual(resp['data']['test'], 'test')
def test_handler_clean(self, mock_unglue):
testRec = {
'httpMethod': 'GET',
'queryStringParameters': {
'isbn': '9999999999'
}
}
mock_unglue().fetchSummary.return_value = {
'status': 200,
'data': {
'test': 'test'
}
}
resp = handler(testRec, None)
self.assertEqual(resp['status'], 200)
self.assertEqual(resp['data']['test'], 'test')
mock_unglue().validate.assert_called_once()
