def test_json_dump():
with open(path_dumps, 'w+') as f:
JsonWrapper.dump(json_json, f)
with open(path_dumps, 'r') as f:
result = JsonWrapper.load(f)
assert json_json == result
if os.path.exists(path_dumps):
os.remove(path_dumps)
def test_schema_fields_order(con, ds):
schema_props_keys = list(
JsonWrapper.loads(
GoogleSheets2DataSource.schema_json())['properties'].keys())
assert schema_props_keys[0] == 'domain'
assert schema_props_keys[1] == 'spreadsheet_id'
assert schema_props_keys[2] == 'sheet'
def test_interpolate_parameters(rok_connector, rok_ds, mocker,
data_source_params, data_source_date_viewid,
query, expected):
"""
check that the query is correctly built with interpolated variables
first case: parameters in the form are set, no parameters in data_source json and placeholders are available in the query
second case: parameters in the form are not set, a parameter exists in data_source json and placeholders are defined in the query
third case: parameters in the form are not set, a parameter exists in data_source json but no placeholder defined in the query
If the conceptor tries a query with placeholders but no matching parameters it will fail, but he is aware of this when using variables in query
"""
mocker.patch.object(RokConnector,
'retrieve_token_with_password',
return_value='fake_authentication_token')
responses.add(
method=responses.POST,
url='https://rok.example.com/graphql?DatabaseName=database',
json={'foo': 'bar'},
)
rok_ds.query = query
rok_ds.start_date = data_source_date_viewid['start_date']
rok_ds.end_date = data_source_date_viewid['end_date']
rok_ds.viewId = data_source_date_viewid['viewId']
rok_ds.parameters = data_source_params
rok_connector.get_df(rok_ds)
for e in expected:
assert e in JsonWrapper.loads(responses.calls[0].request.body)['query']
def get_cache_key(
self,
data_source: Optional[ToucanDataSource] = None,
permissions: Optional[dict] = None,
offset: int = 0,
limit: Optional[int] = None,
) -> str:
"""
Generate a unique identifier (str) for a given connector's configuration
(if no parameters are supplied) or for a given couple connector/query
configuration (if `data_source` parameter is supplied).
This identifier will then be used as a cache key.
"""
unique_identifier = {
'connector': self.get_unique_identifier(),
'permissions': nosql_apply_parameters_to_query(permissions, data_source.parameters)
if data_source
else permissions,
'offset': offset,
'limit': limit,
}
if data_source is not None:
unique_identifier['datasource'] = self._get_unique_datasource_identifier(data_source)
json_uid = JsonWrapper.dumps(unique_identifier, sort_keys=True, default=hash)
string_uid = str(uuid.uuid3(uuid.NAMESPACE_OID, json_uid))
return string_uid
def test_schema_fields_order(connector, create_datasource):
schema_props_keys = list(
JsonWrapper.loads(SoapDataSource.schema_json())['properties'].keys())
assert schema_props_keys[0] == 'domain'
assert schema_props_keys[1] == 'method'
assert schema_props_keys[2] == 'parameters'
assert schema_props_keys[3] == 'flatten_column'
def test_micro_strategy():
js = JsonWrapper.load(open('tests/micro_strategy/fixtures/fixture.json'))
# login
responses.add(
responses.POST,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/auth/login',
headers={'x-mstr-authtoken': 'x'},
status=200,
)
# cube
responses.add(
responses.POST,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/cubes/6137E0964C68D84F107816AA694C2209/instances?limit=100&offset=0', # noqa: E501
json=js,
status=200,
)
df = mc.get_df(md)
assert df.shape == (100, 40)
# report
responses.add(
responses.POST,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/reports/TDjAuKmfGeKnqbxKr1TPfcFr4vBTlWIKDWDvODTSKsQ/instances?limit=100&offset=0', # noqa: E501
json=js,
status=200,
)
df = mc.get_df(mdr)
assert df.shape == (100, 40)
def test_fill_viewfilter_with_ids():
results = JsonWrapper.load(open('tests/micro_strategy/fixtures/fixture.json'))
dfn = get_definition(results)
viewfilter = {
'plop': {'attribute': 'Call Center'},
'plop_id': {'attribute': '8D679D3511D3E4981000E787EC6DE8A4'},
'plip': {'attribute': 'Call Center@DESC'},
'plip_id': {'attribute': '8D679D3511D3E4981000E787EC6DE8A4@DESC'},
'ploup': {'metric': '% Change to Profit'},
'ploup_id': {'metric': '965C42404FD62829356000B0B955F267'},
'poulp': {'constant': 42},
}
res = fill_viewfilter_with_ids(viewfilter, dfn)
assert res['plop'] == {'type': 'attribute', 'id': '8D679D3511D3E4981000E787EC6DE8A4'}
assert res['plop'] == res['plop_id']
assert res['plip'] == {
'type': 'form',
'attribute': {'id': '8D679D3511D3E4981000E787EC6DE8A4'},
'form': {'id': 'CCFBE2A5EADB4F50941FB879CCF1721C'},
}
assert res['plip'] == res['plip_id']
assert res['ploup'] == {'type': 'metric', 'id': '965C42404FD62829356000B0B955F267'}
assert res['ploup'] == res['ploup_id']
assert res['poulp'] == {'type': 'constant', 'dataType': 'Real', 'value': '42'}
def test_retrieve_token_with_password(rok_connector, rok_ds):
"""check that we correctly retrieve the rok token using a passord"""
# This is the data returned by ROK, a token encrypted with the shared secret
auth_query = """
query Auth($database: String!, $user: String!, $password: String!)
{authenticate(database: $database, user: $user, password: $password)}"""
auth_vars = {
'database': rok_ds.database,
'user': rok_connector.username,
'password': rok_connector.password,
}
# Mocks the response we wait from ROK password authentication API
responses.add(
method=responses.POST,
url='http://bla.bla',
json={'data': {
'authenticate': 'rok_token'
}},
)
rok_connector.retrieve_token_with_password(rok_ds.database,
endpoint='http://bla.bla')
assert responses.assert_call_count('http://bla.bla', 1) is True
assert JsonWrapper.loads(responses.calls[0].request.body) == {
'query': auth_query,
'variables': auth_vars,
}
def test_viewfilter():
js = JsonWrapper.load(open('tests/micro_strategy/fixtures/fixture.json'))
expected_viewfilter = {
'operator': 'Equals',
'operands': [
{
'type': 'form',
'attribute': {'id': '8D679D3511D3E4981000E787EC6DE8A4'},
'form': {'id': 'CCFBE2A5EADB4F50941FB879CCF1721C'},
},
{'type': 'constant', 'dataType': 'Char', 'value': 'Miami'},
],
}
# login
responses.add(
responses.POST,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/auth/login',
headers={'x-mstr-authtoken': 'x'},
status=200,
)
# get definition
responses.add(
responses.POST,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/cubes/6137E0964C68D84F107816AA694C2209/instances?limit=0&offset=0', # noqa: E501
json=js,
status=200,
)
# get cube data
responses.add(
responses.POST,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/cubes/6137E0964C68D84F107816AA694C2209/instances?limit=100&offset=0', # noqa: E501
json=js,
status=200,
)
df = mc.get_df(md_filtered)
assert df.shape == (100, 40)
viewfilter = JsonWrapper.loads(responses.calls[2].request.body)['viewFilter']
assert viewfilter == expected_viewfilter
def check_and_feed(host_port):
client = pymongo.MongoClient(
f'mongodb://*****:*****@localhost:{host_port}')
docs_path = f'{os.path.dirname(__file__)}/fixtures/docs.json'
with open(docs_path) as f:
docs_json = f.read()
docs = JsonWrapper.loads(docs_json)
client['toucan']['test_col'].insert_many(docs)
client.close()
def test_fail_retrieve_tokens(oauth2_connector, secrets_keeper):
"""
It should fail ig the stored state does not match the received state
"""
secrets_keeper.save('test',
{'state': JsonWrapper.dumps({'token': 'the_token'})})
with pytest.raises(AssertionError):
oauth2_connector.retrieve_tokens(
f'http://localhost/?state={JsonWrapper.dumps({"token": "bad_token"})}'
)
def get_identifier(self):
json_uid = JsonWrapper.dumps(
{
'name': self.name,
'account': self.account,
'client_id': self.client_id,
'scope': self.scope,
'role': self.role,
},
sort_keys=True,
)
string_uid = str(uuid.uuid3(uuid.NAMESPACE_OID, json_uid))
return string_uid
def retrieve_tokens(self, authorization_response: str, **kwargs):
url = url_parse.urlparse(authorization_response)
url_params = url_parse.parse_qs(url.query)
client = OAuth2Session(
client_id=self.config.client_id,
client_secret=self.config.client_secret.get_secret_value(),
redirect_uri=self.redirect_uri,
)
saved_flow = self.secrets_keeper.load(self.auth_flow_id)
if saved_flow is None:
raise AuthFlowNotFound()
assert (JsonWrapper.loads(
saved_flow['state'])['token'] == JsonWrapper.loads(
url_params['state'][0])['token'])
token = client.fetch_token(
self.token_url,
authorization_response=authorization_response,
client_id=self.config.client_id,
client_secret=self.config.client_secret.get_secret_value(),
**kwargs,
)
self.secrets_keeper.save(self.auth_flow_id, token)
def test_google_analytics(mocker):
gac = GoogleAnalyticsConnector(
type='GoogleAnalytics',
name='Test',
credentials={
'type':
'test',
'project_id':
'test',
'private_key_id':
'test',
'private_key':
'test',
'client_email':
'test',
'client_id':
'test',
'auth_uri':
'https://accounts.google.com/o/oauth2/auth',
'token_uri':
'https://oauth2.googleapis.com/token',
'auth_provider_x509_cert_url':
'https://www.googleapis.com/oauth2/v1/certs',
'client_x509_cert_url':
'https://www.googleapis.com/robot/v1/metadata/x509/pika.com',
},
)
gads = GoogleAnalyticsDataSource(
name='Test',
domain='test',
report_request={
'viewId': '0123456789',
'dateRanges': [{
'startDate': '2018-06-01',
'endDate': '2018-07-01'
}],
},
)
fixture = JsonWrapper.load(
open('tests/google_analytics/fixtures/reports.json'))
module = 'toucan_connectors.google_analytics.google_analytics_connector'
mocker.patch(f'{module}.ServiceAccountCredentials.from_json_keyfile_dict')
mocker.patch(f'{module}.build')
mocker.patch(
f'{module}.get_query_results').return_value = fixture['reports'][0]
df = gac.get_df(gads)
assert df.shape == (3, 11)
def build_authorization_url(self, **kwargs) -> str:
"""Build an authorization request that will be sent to the client."""
client = OAuth2Session(
client_id=self.config.client_id,
client_secret=self.config.client_secret.get_secret_value(),
redirect_uri=self.redirect_uri,
scope=self.scope,
)
state = {'token': generate_token(), **kwargs}
uri, state = client.create_authorization_url(
self.authorization_url, state=JsonWrapper.dumps(state))
self.secrets_keeper.save(self.auth_flow_id, {'state': state})
return uri
def test_retrieve_tokens(mocker, oauth2_connector, secrets_keeper):
"""
It should retrieve tokens and save them
"""
secrets_keeper.save('test',
{'state': JsonWrapper.dumps({'token': 'the_token'})})
mock_fetch_token: Mock = mocker.patch(
'toucan_connectors.oauth2_connector.oauth2connector.OAuth2Session.fetch_token',
return_value={'access_token': 'dummy_token'},
)
oauth2_connector.retrieve_tokens(
f'http://localhost/?state={JsonWrapper.dumps({"token": "the_token"})}')
mock_fetch_token.assert_called()
assert secrets_keeper.load('test')['access_token'] == 'dummy_token'
def test_search():
js = JsonWrapper.load(open('tests/micro_strategy/fixtures/fixture_search.json'))
# login
responses.add(
responses.POST,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/auth/login',
headers={'x-mstr-authtoken': 'x'},
status=200,
)
# search
responses.add(
responses.GET,
'https://demo.microstrategy.com/MicroStrategyLibrary2/api/searches/results?type=776&type=768&offset=0&limit=5&name=revenue+analysis', # noqa: E501
json=js,
status=200,
)
df = mc.get_df(mds)
assert df.shape == (5, 15)
def test_schema_fields_order():
schema_props_keys = list(
JsonWrapper.loads(
SnowflakeConnector.schema_json())['properties'].keys())
ordered_keys = [
'type',
'name',
'account',
'authentication_method',
'user',
'password',
'token_endpoint',
'token_endpoint_content_type',
'role',
'default_warehouse',
'retry_policy',
'secrets_storage_version',
'sso_credentials_keeper',
'user_tokens_keeper',
]
assert schema_props_keys == ordered_keys
def retrieve_data_with_jwt(self, data_source: RokDataSource,
endpoint: str) -> str:
"""Query ROK API with JWT crafted based on the ROK secret to get the Data"""
# Claims defined with ROK
payload = {
'aud': 'Rok-solution',
'iss': 'ToucanToco',
'exp':
str(int((datetime.now() + timedelta(minutes=10)).timestamp())),
'email': self.username,
'iat': str(int(datetime.now().timestamp())),
'nbf': str(int(datetime.now().timestamp())),
}
encoded_payload = encode(payload,
base64.b64decode(self.secret.encode('utf-8')),
algorithm='HS256')
headers = {
'DatabaseName': data_source.database,
'JwtString': encoded_payload,
'Accept': 'application/json',
'Content-Type': 'application/json',
}
try:
res = requests.post(url=endpoint,
data=JsonWrapper.dumps(
{'query': data_source.query}),
headers=headers).json()
except JSONDecodeError:
raise InvalidJWTError('Invalid request, JWT not validated by ROK')
if res.get('Message'):
if 'not authenticated' in res['Message']:
raise InvalidUsernameError('Invalid username')
else:
raise ValueError(res['Message'])
return res
def test_get_df_with_template_overide(data_source, mocker):
co = HttpAPIConnector(
**{
'name': 'test',
'type': 'HttpAPI',
'baseroute': 'http://example.com',
'template': {
'headers': {
'Authorization': 'XX',
'B': '1'
}
},
})
data_source = HttpAPIDataSource(
name='myHttpDataSource',
domain='my_domain',
url='/comments',
json={'A': 1},
headers={'Authorization': 'YY'},
)
responses.add(responses.GET,
'http://example.com/comments',
json=[{
'a': 2
}])
co.get_df(data_source)
h = responses.calls[0].request.headers
j = JsonWrapper.loads(responses.calls[0].request.body)
assert 'Authorization' in h
assert h['Authorization'] == data_source.headers['Authorization']
assert 'B' in h and h['B']
assert 'A' in j and j['A']
def load_file(self) -> dict:
if not path.exists(self.filename):
return {}
with open(self.filename, 'r') as f:
return JsonWrapper.load(f)
import numpy as np
import responses
from toucan_connectors.json_wrapper import JsonWrapper
from toucan_connectors.trello.trello_connector import TrelloConnector, TrelloDataSource
with open('tests/trello/fixtures/fixture.json') as f:
mock_trello_api_json_responses = JsonWrapper.load(f)
trello_connector = TrelloConnector(name='trello')
baseroute = 'https://api.trello.com/1/boards/dsjhdejbdkeb'
default_param = 'key=&token=&'
@responses.activate
def test_get_board_method():
responses.add(
responses.GET,
f'{baseroute}/lists?fields=name',
json=mock_trello_api_json_responses['lists'],
status=200,
)
lists = trello_connector.get_board(fields='name', path='dsjhdejbdkeb/lists')
assert len(lists) == 2
assert set(lists[0].keys()) == {'id', 'name'}
assert set(lists[0].values()) == {'5b2775500401ad42967638a8', 'zorro'}
data_result_none = []
data_result_one = [{
'1 Column Name': 'value',
'2 Column Name': 'value',
'3 Column Name': 'value',
'4 Column Name': 'value',
'5 Column Name': 'value',
'6 Column Name': 'value',
'7 Column Name': 'value',
'8 Column Name': 'value',
'9 Column Name': 'value',
'10 Column Name': 'value',
'11 Column Name': 'value',
}]
data_result_5 = JsonWrapper.load(
open('tests/fixtures/fixture_snowflake_common/data_5.json', ))
data_result_all = JsonWrapper.load(
open('tests/fixtures/fixture_snowflake_common/data_10.json', ))
databases_result_all = [{'name': 'database_1'}, {'name': 'database_2'}]
databases_result_none = []
databases_result_one = [{'name': 'database_1'}]
warehouses_result_all = [{'name': 'warehouse_1'}, {'name': 'warehouse_2'}]
warehouses_result_none = []
warehouses_result_one = [{'name': 'warehouse_1'}]
@patch('snowflake.connector.connect',
return_value=snowflake.connector.SnowflakeConnection)
@patch('snowflake.connector.cursor.SnowflakeCursor.execute', return_value=None)
@patch('pandas.DataFrame.from_dict',
return_value=pd.DataFrame(databases_result_all))
def test_json_loads_not_valid():
with pytest.raises(JSONDecodeError):
JsonWrapper.loads(json_not_valid_string)
def test_json_loads():
result = JsonWrapper.loads(json_string)
assert json_json == result
database='database_1',
warehouse='warehouse_1',
query='test_query with %(foo)s and %(pokemon)s',
query_object={
'schema': 'SHOW_SCHEMA',
'table': 'MY_TABLE',
'columns': ['col1', 'col2']
},
parameters={
'foo': 'bar',
'pokemon': 'pikachu'
},
)
data = JsonWrapper.load(open('tests/snowflake/fixture/data.json', ))
df = pd.DataFrame(
data,
columns=[
'1 Column Name',
'2 Column Name',
'3 Column Name',
'4 Column Name',
'5 Column Name',
'6 Column Name',
'7 Column Name',
'8 Column Name',
'9 Column Name',
'10 Column Name',
],
)
def test_json_loads_not_string():
with pytest.raises(TypeError):
JsonWrapper.loads(json_not_string)
def test_json_load():
result = JsonWrapper.load(open(path_loads, 'r'))
assert {'key1': 'value1', 'key2': 'value2'} == result
def test_json_load_file_not_found():
with pytest.raises(FileNotFoundError):
JsonWrapper.load(open(path_not_found, 'r'))
def save(self, key: str, value):
values = self.load_file()
values[key] = value
with open(self.filename, 'w') as f:
JsonWrapper.dump(values, f)