def test_oauth2_oidc_authentication(mocker):
data_provider = {
'type': 'HttpAPI',
'name': 'bidule-api',
'baseroute': 'https://api.bidule.com',
'auth': {
'type': 'oauth2_oidc',
'args': [],
'kwargs': {
'id_token': 'id_token_test',
'refresh_token': 'refresh_tokenètest',
'client_id': 'provided_client_id',
'client_secret': 'provided_client_secret',
'token_endpoint': 'https://api.bidule.com/token',
},
},
}
data_source = {
'domain': 'test',
'name': 'bidule-api',
'url': '/data',
'parameters': {
'some': 'variable'
},
'method': 'GET',
}
c = HttpAPIConnector(**data_provider)
session = requests.Session()
session.headers.update({'Authorization': 'Bearer MyNiceToken'})
mock_session = mocker.patch('toucan_connectors.auth.oauth2_oidc')
responses.add(method=responses.GET,
url='https://api.bidule.com/data',
json={'ultimecia': 'citadel'})
c.get_df(HttpAPIDataSource(**data_source))
mock_session.assert_called_once()
def test_get_df_oauth2_backend_mocked():
data_provider = {
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'https://gateway.eu1.mindsphere.io/api/im/v3',
'auth': {
'type':
'oauth2_backend',
'args': [
'https://mscenter.piam.eu1.mindsphere.io/oauth/token',
'<client_id>',
'<client_secret>',
],
},
}
users = {'domain': 'test', 'name': 'test', 'url': '/Users'}
responses.add(
responses.POST,
'https://mscenter.piam.eu1.mindsphere.io/oauth/token',
json={'access_token': 'A'},
)
responses.add(responses.GET,
'https://gateway.eu1.mindsphere.io/api/im/v3/Users',
json=[{
'A': 1
}])
co = HttpAPIConnector(**data_provider)
co.get_df(HttpAPIDataSource(**users))
assert len(responses.calls) == 2
def test_no_top_level_domain():
data_provider = {
'name': 'DataServiceApi',
'type': 'HttpAPI',
# before we relaxed the type of baseroute using AnyHttpUrl
# this "domain" would trigger a validation error
'baseroute': 'http://cd-arggh-v2:9088/api/aggregations/v1/',
}
c = HttpAPIConnector(**data_provider)
# we want to check as well that we call the right urls
data_source = {
'domain': 'appendable3_id_9001',
'name': 'DataServiceApi',
'url': 'forecast_90_days/site/%(SITE_VARIABLE)s/date/%(REQ_DATE)s',
'parameters': {
'SITE_VARIABLE': 'blah',
'REQ_DATE': '123456'
},
'method': 'GET',
}
responses.add(
responses.GET,
'http://cd-arggh-v2:9088/api/aggregations/v1/forecast_90_days/site/blah/date/123456',
json=[{
'a': 1
}],
)
c.get_df(HttpAPIDataSource(**data_source))
assert len(responses.calls) == 1
def test_get_df_with_template_overide(data_source, mocker):
co = HttpAPIConnector(
**{
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'http://example.com',
'template': {
'headers': {
'Authorization': 'XX',
'B': '1'
}
}
})
data_source.headers = {'Authorization': 'YY'}
data_source.json = {'A': 1}
responses.add(responses.GET,
'http://example.com/comments',
json=[{
"a": 2
}])
co.get_df(data_source)
h = responses.calls[0].request.headers
j = json.loads(responses.calls[0].request.body)
assert 'Authorization' in h
assert h['Authorization'] == data_source.headers['Authorization']
assert 'B' in h and h['B']
assert 'A' in j and j['A']
def test_exceptions_not_json():
connector = HttpAPIConnector(name='myHttpConnector',
type='HttpAPI',
baseroute='https://demo.toucantoco.com')
data_source = HttpAPIDataSource(name='myHttpDataSource',
domain='my_domain',
url='/')
with pytest.raises(ValueError):
connector.get_df(data_source)
def test_exceptions_not_json():
connector = HttpAPIConnector(name="myHttpConnector",
type="HttpAPI",
baseroute="https://demo.toucantoco.com")
data_source = HttpAPIDataSource(name="myHttpDataSource",
domain="my_domain",
url="/")
with pytest.raises(ValueError):
connector.get_df(data_source)
def test_get_df_oauth2_backend():
data_provider = {
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'https://gateway.eu1.mindsphere.io/api/im/v3',
'auth': {
'type':
'oauth2_backend',
'args': [
'https://mscenter.piam.eu1.mindsphere.io/oauth/token',
'<client_id>',
'<client_secret>',
],
},
}
users = {
'domain': 'test',
'name': 'test',
'url': '/Users',
'filter': '.resources'
}
co = HttpAPIConnector(**data_provider)
df = co.get_df(HttpAPIDataSource(**users))
assert 'userName' in df
def test_get_df_with_template(data_source, mocker):
co = HttpAPIConnector(
**{
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'http://example.com',
'template': {
'headers': {
'Authorization': 'XX'
}
},
})
responses.add(responses.GET,
'http://example.com/comments',
json=[{
'a': 2
}])
co.get_df(data_source)
h = responses.calls[0].request.headers
assert 'Authorization' in h
assert h['Authorization'] == co.template.headers['Authorization']
def test_get_df_with_template_overide(data_source, mocker):
co = HttpAPIConnector(
**{
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'http://example.com',
'template': {
'headers': {
'Authorization': 'XX',
'B': '1'
}
},
})
data_source = HttpAPIDataSource(
name='myHttpDataSource',
domain='my_domain',
url='/comments',
json={'A': 1},
headers={'Authorization': 'YY'},
)
responses.add(responses.GET,
'http://example.com/comments',
json=[{
'a': 2
}])
co.get_df(data_source)
h = responses.calls[0].request.headers
j = JsonWrapper.loads(responses.calls[0].request.body)
assert 'Authorization' in h
assert h['Authorization'] == data_source.headers['Authorization']
assert 'B' in h and h['B']
assert 'A' in j and j['A']
def test_e2e():
con_params = {
'name': 'open_data_paris',
'baseroute': 'https://opendata.paris.fr/api/'
}
ds_params = {
'domain': 'books',
'name': 'open_data_paris',
'url': 'records/1.0/search/',
'params': {
'dataset':
'les-1000-titres-les-plus-reserves-dans-les-bibliotheques-de-pret',
'facet': 'auteur',
'sort': 'rang',
'rows': 1000,
},
'filter': '.records[].fields',
}
con = HttpAPIConnector(**con_params)
ds = HttpAPIDataSource(**ds_params)
df = con.get_df(ds)
assert df.shape == (1000, 5)