def test_existing_gf_id(event, obj): """ Test that a new genomic file is created with predefined kf_id """ os.environ['DATASERVICE_API'] = 'http://api.com/' obj() s3 = boto3.client('s3') # Add a gf_id tags = TAGS.copy() tags['TagSet'].append({'Key': 'gf_id', 'Value': 'GF_00000002'}) response = s3.put_object_tagging(Bucket=BUCKET, Key=OBJECT, Tagging=tags) mock = patch('service.requests') req = mock.start() def mock_get(url, *args, **kwargs): if '/genomic-files' in url: resp = MagicMock() resp.status_code = 404 return resp elif '/biospecimens' in url: resp = MagicMock() resp.json.return_value = {'results': {'kf_id': url[:-11]}} resp.status_code = 200 return resp elif '/studies' in url: resp = MagicMock() resp.status_code = 200 return resp req.get.side_effect = mock_get mock_resp = MagicMock() mock_resp.json.return_value = {'results': {'kf_id': 'GF_00000000'}} mock_resp.status_code = 201 req.post.return_value = mock_resp service.handler(event, {}) expected = { 'kf_id': 'GF_00000002', 'file_name': '60d33dec-98db-446c-ac64-f4d027588f26.cram', 'file_format': 'cram', 'acl': ['SD_9PYZAHHE'], 'data_type': 'Aligned Reads', 'controlled_access': True, 'availability': 'Immediate Download', 'is_harmonized': True, 'reference_genome': 'GRCh38', 'biospecimen_id': 'BS_QV3Z0DZM', 'hashes': { 'etag': 'd41d8cd98f00b204e9800998ecf8427e' }, 'size': 1024, 'urls': ['s3://{}/{}'.format(BUCKET, OBJECT)] } assert req.get.call_count == 4 req.post.assert_any_call('http://api.com/genomic-files', json=expected) assert req.post.call_count == 2 mock.stop()
def test_api_key_set(monkeypatch): """ Ensures exception raised if api key is not set """ monkeypatch.delenv("API_KEY", raising=False) event = dict(freshen_url="doesn't matter") with pytest.raises(KeyError): handler(event, None)
def test_records_none(self): testRec = {'source': 'Kinesis'} try: handler(testRec, None) except NoRecordsReceived: pass self.assertRaises(NoRecordsReceived)
def test_out_of_time(event, obj): """ Test that a function is re-invoked when records remain """ obj() os.environ['DATASERVICE_API'] = 'http://api.com/' mock_r = patch('service.requests') req = mock_r.start() class Context: def __init__(self): self.invoked_function_arn = 'arn:aws:lambda:::function:kf-lambda' def get_remaining_time_in_millis(self): return 300 # Add a second record event['Records'].append(event['Records'][0]) with patch('service.boto3.client') as mock: service.handler(event, Context()) assert mock().invoke.call_count == 1 _, args = mock().invoke.call_args_list[0] assert args['FunctionName'] == Context().invoked_function_arn assert args['InvocationType'] == 'Event' payload = json.loads(args['Payload'].decode('utf-8')) assert payload == {'Records': [event['Records'][1]]} mock_r.stop()
def test_handler_error(self): testRec = {'source': 'Kinesis', 'Records': []} try: handler(testRec, None) except NoRecordsReceived: pass self.assertRaises(NoRecordsReceived)
def test_upload(obj, messages, req, env): """ Test that compressed log file is created """ s3 = obj() service.handler({}, Context(40000)) assert len(s3.list_objects(Bucket=BUCKET)['Contents']) == 1 stream = io.BytesIO() KEY = s3.list_objects(Bucket=BUCKET)['Contents'][0]['Key'] ob = s3.get_object(Bucket=BUCKET, Key=KEY) #dat = gzip.decompress(ob['Body'].read()).decode() dat = ob['Body'].read().decode() for i, l in enumerate(dat.split('\n')): assert json.loads(messages[i])['Message'].replace("'", '"') == l message = json.loads(l) assert 'path' in message assert 'method' in message assert 'api_version' in message assert 'api_commit' in message assert 'data' in message assert req.post.call_count == 2
def test_recent_adoptions(self): data = { "username": "******", "password": "******", "method": "xml_recent_adoptions" } post = utils.PostedData(data, "en") service.handler(post, "127.0.0.1", "sheltermanager.com") data["method"] = "json_recent_adoptions" service.handler(post, "127.0.0.1", "sheltermanager.com")
def test_handler_error(self): testRec = { 'httpMethod': 'GET', 'queryStringParameters': { 'nonISBN': '987.23' } } try: handler(testRec, None) except InvalidExecutionType: pass self.assertRaises(InvalidExecutionType)
def test_handler_error(self): testRec = { 'httpMethod': 'GET', 'queryStringParameters': { 'otherQuery': 'Tester, Test' } } try: handler(testRec, None) except InvalidExecutionType: pass self.assertRaises(InvalidExecutionType)
def test_delete(obj, messages, queue, req, env): """ Test that messages are deleted after being written """ s3 = obj() queue_url = queue() service.handler({}, Context(40000)) sqs = boto3.resource('sqs', region_name='us-east-1') queue = sqs.Queue(queue_url) batch = queue.receive_messages(MaxNumberOfMessages=10, WaitTimeSeconds=0) assert len(batch) == 0 assert req.post.call_count == 2
def test_query_filter_mapping(monkeypatch): monkeypatch.setenv("BITLY_GENERIC_ACCESS_TOKEN", "my_token") url = dict(url="https://arxiv.org/") event = dict(queryStringParameters=url) result_dict = handler(event, None) expected_dict = {'statusCode': 200, 'body': '{"id": "https://j.mp/2soOxSO"}'} assert result_dict == expected_dict
def test_handler_empty(self, mock_fetch): testRec = { 'source': 'Kinesis', 'Records': [{'some': 'record'}] } resp = handler(testRec, None) mock_fetch.assert_called_once() self.assertEqual(resp[0][0], 'empty')
def test_handler_handles_http_error(mocker, event): """Ensures any HTTP client errors get handled""" mocker.patch( "service.urlopen", side_effect=HTTPError("http://example.com", 500, "Internal Error", {}, None), ) assert handler(event, None) is False
def test_handler_clean(self, mocker): with patch.multiple(SourceManager, fetchRecords=DEFAULT, sendWorksToKinesis=DEFAULT) as managerMocks: outWorks = handler({}, {}) managerMocks['fetchRecords'].assert_called_once() managerMocks['sendWorksToKinesis'].assert_called_once() assert outWorks == []
def test_handler_raises(monkeypatch): """Reads from the `test_handler_raises` cassette and processes the request that raises """ monkeypatch.setenv("API_KEY", "mykey") event = dict( freshen_url= "https://hostdoesnotexist.gov/v1/some_endpoint/freshen.json?api_key=") resp = handler(event, None) assert resp is False
def test_reinvoke(obj, messages, queue, req, env): """ Test that messages are deleted after being written """ s3 = obj() queue_url = queue() patch_lam = patch('boto3.client') mock_lam = patch_lam.start() service.handler({}, Context(2000)) sqs = boto3.resource('sqs', region_name='us-east-1') queue = sqs.Queue(queue_url) assert mock_lam().invoke.call_count == 1 assert mock_lam().put_object.call_count == 1 assert req.post.call_count == 2 mock_lam.stop()
def test_handler(monkeypatch): """Reads from the `test_handler` cassette and processes the request """ monkeypatch.setenv("API_KEY", "mykey") event = dict( freshen_url= "https://api.trade.gov/v1/some_endpoint/freshen.json?api_key=") resp = handler(event, None) assert resp is True
def test_handler_local(self, mock_parser, mock_load): testRec = { 'source': 'local.file', 'localFile': 'test.csv' } resp = handler(testRec, None) mock_load.assert_called_once() mock_parser.assert_called_once() self.assertTrue(resp)
def test_handler(): """ Test the service handler """ r_patch = patch('service.requests') r_mock = r_patch.start() mock_resp = Mock() mock_resp.status_code = 200 r_mock.get.return_value = mock_resp event = { 'service': 'http://kids-first.github.io', 'endpoint': '/' } service.handler(event, {}) assert r_mock.get.call_count == 1 r_mock.get.assert_called_with('http://kids-first.github.io/', headers={'User-Agent': 'HealthChecker'})
def test_handler_scheduled(self, mock_parser, mock_fetch): testRec = { 'source': 'Kinesis', 'Records': [{'some': 'record'}] } resp = handler(testRec, None) mock_fetch.assert_called_once() mock_parser.assert_called_once() self.assertTrue(resp)
def test_handler_clean(self, mockResponse, mockFetch): testRec = { 'queryStringParameters': { 'identifier': '000000000', 'type': 'oclc' } } resp = handler(testRec, None) self.assertEqual(resp, 'outObject') mockFetch.assert_called_once_with('000000000', 'oclc') mockResponse.assert_called_once_with(200, 'oclcResponse')
def test_handler_clean(self): testRec = { 'source': 'Kinesis', 'Records': [{ 'kinesis': { 'data': 'data' } }] } resp = handler(testRec, None) self.assertEqual(resp, 'Hello, World')
def test_handler_internal_error(self, mockResponse, mockFetch): testRec = { 'queryStringParameters': { 'identifier': '000000000', 'type': 'oclc' } } resp = handler(testRec, None) self.assertEqual(resp, 'outObject') mockFetch.assert_called_once_with('000000000', 'oclc') mockResponse.assert_called_once_with(500, {'message': 'Test Error'})
def test_handler_clean(self, mock_parse): testRec = { 'source': 'Kinesis', 'Records': [{ 'kinesis': { 'data': 'data' } }] } resp = handler(testRec, None) mock_parse.assert_called_once()
def test_handler_local(self, mock_parser, mock_load): testRec = { 'source': 'local.file', 'localFile': 'test.csv', 'start': 0, 'size': 1 } resp = handler(testRec, None) mock_load.assert_called_once() mock_parser.assert_called_once() self.assertEqual(resp, [1, 2])
def test_handler_error_bad_parameters(self, mockResponse): testRec = { 'queryStringParameters': { 'identifier': '000000000', } } resp = handler(testRec, None) self.assertEqual(resp, 'outObject') mockResponse.assert_called_once_with( 400, {'message': 'GET query must include identifier and type'} )
def test_viaf_query_error(self, mock_viaf): testRec = { 'httpMethod': 'GET', 'queryStringParameters': { 'queryName': 'Tester, Test' } } mock_viaf().query.side_effect = VIAFError('test viaf error') mock_viaf.formatResponse.return_value = { 'status': 500, 'data': 'test error' } resp = handler(testRec, None) self.assertEqual(resp['status'], 500)
def test_existing_gf(event, obj): """ Test that a new genomic file is not created if it exists """ obj() s3 = boto3.client('s3') # Add a gf_id tags = TAGS.copy() tags['TagSet'].append({'Key': 'gf_id', 'Value': 'GF_00000001'}) response = s3.put_object_tagging(Bucket=BUCKET, Key=OBJECT, Tagging=tags) os.environ['DATASERVICE_API'] = 'http://api.com/' mock = patch('service.requests') req = mock.start() mock_resp = MagicMock() mock_resp.json.return_value = {'results': {'kf_id': 'GF_00000001'}} mock_resp.status_code = 200 req.get.return_value = mock_resp service.handler(event, {}) assert req.get.call_count == 1 assert req.post.call_count == 0 mock.stop()
def test_unglue_query_error(self, mock_unglue): testRec = { 'httpMethod': 'GET', 'queryStringParameters': { 'isbn': '9999999999' } } mock_unglue().fetchSummary.side_effect = UnglueError( 500, 'test unglue error') mock_unglue.formatResponse.return_value = { 'status': 500, 'data': 'test error' } resp = handler(testRec, None) self.assertEqual(resp['status'], 500)
def test_handler_clean(self, mock_viaf): testRec = { 'httpMethod': 'GET', 'queryStringParameters': { 'queryName': 'Tester, Test' } } mock_viaf().query.return_value = { 'status': 200, 'data': { 'test': 'test' } } resp = handler(testRec, None) self.assertEqual(resp['status'], 200) self.assertEqual(resp['data']['test'], 'test')
def test_handler_clean(self, mock_unglue): testRec = { 'httpMethod': 'GET', 'queryStringParameters': { 'isbn': '9999999999' } } mock_unglue().fetchSummary.return_value = { 'status': 200, 'data': { 'test': 'test' } } resp = handler(testRec, None) self.assertEqual(resp['status'], 200) self.assertEqual(resp['data']['test'], 'test') mock_unglue().validate.assert_called_once()