def test_oauth2_oidc_authentication(mocker): data_provider = { 'type': 'HttpAPI', 'name': 'bidule-api', 'baseroute': 'https://api.bidule.com', 'auth': { 'type': 'oauth2_oidc', 'args': [], 'kwargs': { 'id_token': 'id_token_test', 'refresh_token': 'refresh_tokenètest', 'client_id': 'provided_client_id', 'client_secret': 'provided_client_secret', 'token_endpoint': 'https://api.bidule.com/token', }, }, } data_source = { 'domain': 'test', 'name': 'bidule-api', 'url': '/data', 'parameters': { 'some': 'variable' }, 'method': 'GET', } c = HttpAPIConnector(**data_provider) session = requests.Session() session.headers.update({'Authorization': 'Bearer MyNiceToken'}) mock_session = mocker.patch('toucan_connectors.auth.oauth2_oidc') responses.add(method=responses.GET, url='https://api.bidule.com/data', json={'ultimecia': 'citadel'}) c.get_df(HttpAPIDataSource(**data_source)) mock_session.assert_called_once()
def test_no_top_level_domain(): data_provider = { 'name': 'DataServiceApi', 'type': 'HttpAPI', # before we relaxed the type of baseroute using AnyHttpUrl # this "domain" would trigger a validation error 'baseroute': 'http://cd-arggh-v2:9088/api/aggregations/v1/', } c = HttpAPIConnector(**data_provider) # we want to check as well that we call the right urls data_source = { 'domain': 'appendable3_id_9001', 'name': 'DataServiceApi', 'url': 'forecast_90_days/site/%(SITE_VARIABLE)s/date/%(REQ_DATE)s', 'parameters': { 'SITE_VARIABLE': 'blah', 'REQ_DATE': '123456' }, 'method': 'GET', } responses.add( responses.GET, 'http://cd-arggh-v2:9088/api/aggregations/v1/forecast_90_days/site/blah/date/123456', json=[{ 'a': 1 }], ) c.get_df(HttpAPIDataSource(**data_source)) assert len(responses.calls) == 1
def test_get_df_oauth2_backend_mocked(): data_provider = { 'name': 'test', 'type': 'HttpAPI', 'baseroute': 'https://gateway.eu1.mindsphere.io/api/im/v3', 'auth': { 'type': 'oauth2_backend', 'args': [ 'https://mscenter.piam.eu1.mindsphere.io/oauth/token', '<client_id>', '<client_secret>', ], }, } users = {'domain': 'test', 'name': 'test', 'url': '/Users'} responses.add( responses.POST, 'https://mscenter.piam.eu1.mindsphere.io/oauth/token', json={'access_token': 'A'}, ) responses.add(responses.GET, 'https://gateway.eu1.mindsphere.io/api/im/v3/Users', json=[{ 'A': 1 }]) co = HttpAPIConnector(**data_provider) co.get_df(HttpAPIDataSource(**users)) assert len(responses.calls) == 2
def test_get_df_with_template_overide(data_source, mocker): co = HttpAPIConnector( **{ 'name': 'test', 'type': 'HttpAPI', 'baseroute': 'http://example.com', 'template': { 'headers': { 'Authorization': 'XX', 'B': '1' } } }) data_source = HttpAPIDataSource(name="myHttpDataSource", domain="my_domain", url="/comments", json={'A': 1}, headers={'Authorization': 'YY'}) responses.add(responses.GET, 'http://example.com/comments', json=[{ "a": 2 }]) co.get_df(data_source) h = responses.calls[0].request.headers j = json.loads(responses.calls[0].request.body) assert 'Authorization' in h assert h['Authorization'] == data_source.headers['Authorization'] assert 'B' in h and h['B'] assert 'A' in j and j['A']
def test_get_df_oauth2_backend(): data_provider = { 'name': 'test', 'type': 'HttpAPI', 'baseroute': 'https://gateway.eu1.mindsphere.io/api/im/v3', 'auth': { 'type': 'oauth2_backend', 'args': [ 'https://mscenter.piam.eu1.mindsphere.io/oauth/token', '<client_id>', '<client_secret>', ], }, } users = { 'domain': 'test', 'name': 'test', 'url': '/Users', 'filter': '.resources' } co = HttpAPIConnector(**data_provider) df = co.get_df(HttpAPIDataSource(**users)) assert 'userName' in df
def test_schema_extra(): data_source_spec = { 'data': '', 'domain': 'Clickhouse test', 'filter': '', 'flatten_column': '', 'headers': {}, 'json': {}, 'live_data': False, 'load': True, 'method': 'GET', 'name': 'Some clickhouse provider', 'parameters': { 'ids': [3986, 3958] }, 'params': {}, 'proxies': {}, 'type': 'external_database', 'url': '', 'validation': {}, 'xpath': '', } conf = HttpAPIDataSource(**data_source_spec).Config schema = { 'properties': { 'data': '', 'proxies': {}, 'filter': '', 'flatten_column': '', 'validation': {}, 'xpath': '', } } conf.schema_extra(schema, model=HttpAPIDataSource) assert schema == { 'properties': { 'proxies': {}, 'flatten_column': '', 'data': '', 'xpath': '', 'filter': '', 'validation': {}, } }
def xml_datasource(): data_source = { 'domain': 'testxml', 'name': 'XMLAPI', 'url': 'foo/xml', 'method': 'GET', 'xpath': 'output', 'filter': '.output.users.user', } return HttpAPIDataSource(**data_source)
def test_exceptions_not_json(): connector = HttpAPIConnector(name='myHttpConnector', type='HttpAPI', baseroute='https://demo.toucantoco.com') data_source = HttpAPIDataSource(name='myHttpDataSource', domain='my_domain', url='/') with pytest.raises(ValueError): connector.get_df(data_source)
def test_exceptions_not_json(): connector = HttpAPIConnector(name="myHttpConnector", type="HttpAPI", baseroute="https://demo.toucantoco.com") data_source = HttpAPIDataSource(name="myHttpDataSource", domain="my_domain", url="/") with pytest.raises(ValueError): connector.get_df(data_source)
def test_get_df_with_json(connector, data_source, mocker): data_source = HttpAPIDataSource(name='myHttpDataSource', domain='my_domain', url='/comments', json={'a': 1}) responses.add(responses.GET, 'https://jsonplaceholder.typicode.com/comments', json=[{ 'a': 2 }]) connector.get_df(data_source) assert responses.calls[0].request.body == b'{"a": 1}'
def test_e2e(): con_params = { 'name': 'open_data_paris', 'baseroute': 'https://opendata.paris.fr/api/' } ds_params = { 'domain': 'books', 'name': 'open_data_paris', 'url': 'records/1.0/search/', 'params': { 'dataset': 'les-1000-titres-les-plus-reserves-dans-les-bibliotheques-de-pret', 'facet': 'auteur', 'sort': 'rang', 'rows': 1000, }, 'filter': '.records[].fields', } con = HttpAPIConnector(**con_params) ds = HttpAPIDataSource(**ds_params) df = con.get_df(ds) assert df.shape == (1000, 5)
def test_get_df_with_json_flatten_column(connector, data_source, mocker): data_source = HttpAPIDataSource( name='myHttpDataSource', domain='my_domain', url='/comments', flatten_column='products', ) responses.add( responses.GET, 'https://jsonplaceholder.typicode.com/comments', json={ 'brand': 'brewdog', 'products': [{ 'name': 'punk' }, { 'name': '5pm' }] }, ) result = connector.get_df(data_source) assert list(result['products.name']) == ['punk', '5pm']
def test_with_proxies(mocker): req = mocker.patch( 'toucan_connectors.http_api.http_api_connector.Session.request') f = 'toucan_connectors.http_api.http_api_connector.transform_with_jq' mocker.patch(f).return_value = [{'a': 1}] data_provider = { 'name': 'test', 'type': 'HttpApi', 'baseroute': 'https://example.com' } data_source = { 'proxies': { 'https': 'https://eu1.proxysite.com' }, 'name': 'test', 'domain': 'test_domain', 'url': '/endpoint' } HttpAPIConnector(**data_provider).get_df(HttpAPIDataSource(**data_source)) 'proxies' in req.call_args[1]
def test_with_cert(mocker): req = mocker.patch( 'toucan_connectors.http_api.http_api_connector.Session.request') f = 'toucan_connectors.http_api.http_api_connector.transform_with_jq' mocker.patch(f).return_value = [{'a': 1}] data_provider = { 'name': 'test', 'type': 'HttpApi', 'baseroute': 'https://example.com', 'cert': ['tests/http_api/test_http_api.py', 'tests/http_api/test_http_api.py'], } data_source = {'name': 'test', 'domain': 'test_domain', 'url': '/endpoint'} HttpAPIConnector(**data_provider).get_df(HttpAPIDataSource(**data_source)) args, kwargs = req.call_args assert kwargs['cert'] == [ 'tests/http_api/test_http_api.py', 'tests/http_api/test_http_api.py' ]
def data_source(): return HttpAPIDataSource(name='myHttpDataSource', domain='my_domain', url='/comments')
def data_source(): return HttpAPIDataSource(name="myHttpDataSource", domain="my_domain", url="/comments")