def test_e2e():
con_params = {
'name': 'open_data_paris',
'baseroute': 'https://opendata.paris.fr/api/'
}
ds_params = {
'domain': 'books',
'name': 'open_data_paris',
'url': 'records/1.0/search/',
'params': {
'dataset':
'les-1000-titres-les-plus-reserves-dans-les-bibliotheques-de-pret',
'facet': 'auteur',
'sort': 'rang',
'rows': 1000,
},
'filter': '.records[].fields',
}
con = HttpAPIConnector(**con_params)
ds = HttpAPIDataSource(**ds_params)
df = con.get_df(ds)
assert df.shape == (1000, 5)
def test_get_df_with_template_overide(data_source, mocker):
co = HttpAPIConnector(
**{
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'http://example.com',
'template': {
'headers': {
'Authorization': 'XX',
'B': '1'
}
},
})
data_source = HttpAPIDataSource(
name='myHttpDataSource',
domain='my_domain',
url='/comments',
json={'A': 1},
headers={'Authorization': 'YY'},
)
responses.add(responses.GET,
'http://example.com/comments',
json=[{
'a': 2
}])
co.get_df(data_source)
h = responses.calls[0].request.headers
j = json.loads(responses.calls[0].request.body)
assert 'Authorization' in h
assert h['Authorization'] == data_source.headers['Authorization']
assert 'B' in h and h['B']
assert 'A' in j and j['A']
def test_get_df_with_template(data_source, mocker):
co = HttpAPIConnector(
**{
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'http://example.com',
'template': {
'headers': {
'Authorization': 'XX'
}
},
})
responses.add(responses.GET,
'http://example.com/comments',
json=[{
'a': 2
}])
co.get_df(data_source)
h = responses.calls[0].request.headers
assert 'Authorization' in h
assert h['Authorization'] == co.template.headers['Authorization']
def test_with_proxies(mocker):
req = mocker.patch(
'toucan_connectors.http_api.http_api_connector.Session.request')
f = 'toucan_connectors.http_api.http_api_connector.transform_with_jq'
mocker.patch(f).return_value = [{'a': 1}]
data_provider = {
'name': 'test',
'type': 'HttpApi',
'baseroute': 'https://example.com'
}
data_source = {
'proxies': {
'https': 'https://eu1.proxysite.com'
},
'name': 'test',
'domain': 'test_domain',
'url': '/endpoint'
}
HttpAPIConnector(**data_provider).get_df(HttpAPIDataSource(**data_source))
'proxies' in req.call_args[1]
def test_with_cert(mocker):
req = mocker.patch(
'toucan_connectors.http_api.http_api_connector.Session.request')
f = 'toucan_connectors.http_api.http_api_connector.transform_with_jq'
mocker.patch(f).return_value = [{'a': 1}]
data_provider = {
'name':
'test',
'type':
'HttpApi',
'baseroute':
'https://example.com',
'cert':
['tests/http_api/test_http_api.py', 'tests/http_api/test_http_api.py'],
}
data_source = {'name': 'test', 'domain': 'test_domain', 'url': '/endpoint'}
HttpAPIConnector(**data_provider).get_df(HttpAPIDataSource(**data_source))
args, kwargs = req.call_args
assert kwargs['cert'] == [
'tests/http_api/test_http_api.py', 'tests/http_api/test_http_api.py'
]
def connector():
return HttpAPIConnector(name='myHttpConnector',
type='HttpAPI',
baseroute='https://jsonplaceholder.typicode.com')
def connector():
return HttpAPIConnector(name="myHttpConnector",
type="HttpAPI",
baseroute="https://jsonplaceholder.typicode.com")