def test_uri():
connector = MongoConnector(name='my_mongo_con',
host='myhost',
port='123',
database='mydb')
assert connector.uri == 'mongodb://*****:*****@myhost:123'
connector = MongoConnector(name='my_mongo_con',
host='myhost',
port='123',
database='mydb',
username='******',
password='******')
assert connector.uri == 'mongodb://*****:*****@myhost:123'
with pytest.raises(ValueError) as exc_info:
MongoConnector(name='my_mongo_con',
host='myhost',
port='123',
database='mydb',
password='******')
assert 'password:\n username must be set' in str(exc_info.value)
def mongo_connector(mongo_server):
return MongoConnector(name='mycon',
host='localhost',
database='toucan',
port=mongo_server['port'],
username='******',
password='******')
def test_client_args_with_mongo_uri(mocker):
"""It should not pass any other parameter than the host to MongoClient"""
mongo_client_mock = mocker.patch(
'toucan_connectors.mongo.mongo_connector.pymongo.MongoClient')
connector = MongoConnector(name='my_mongo_con',
host='mongodb://*****:*****@myhost:123')
connector.client
mongo_client_mock.assert_called_with(
host='mongodb://*****:*****@myhost:123')
def test_get_df(mocker):
class MongoMock:
def __init__(self, database, collection):
self.data = {database: {collection: pymongo.collection.Collection}}
def __getitem__(self, row):
return self.data[row]
def close(self):
pass
snock = mocker.patch('pymongo.MongoClient')
snock.return_value = MongoMock('toucan', 'test_col')
find = mocker.patch('pymongo.collection.Collection.find')
aggregate = mocker.patch('pymongo.collection.Collection.aggregate')
mongo_connector = MongoConnector(name='mycon',
host='localhost',
database='toucan',
port=22,
username='******',
password='******')
datasource = MongoDataSource(name='mycon',
domain='mydomain',
collection='test_col',
query={'domain': 'domain1'})
mongo_connector.get_df(datasource)
datasource = MongoDataSource(name='mycon',
domain='mydomain',
collection='test_col',
query='domain1')
mongo_connector.get_df(datasource)
datasource = MongoDataSource(name='mycon',
domain='mydomain',
collection='test_col',
query=[{
'$match': {
'domain': 'domain1'
}
}])
mongo_connector.get_df(datasource)
snock.assert_called_with('mongodb://*****:*****@localhost:22',
ssl=False)
assert snock.call_count == 3
find.assert_called_with({'domain': 'domain1'})
assert find.call_count == 2
aggregate.assert_called_once_with([{'$match': {'domain': 'domain1'}}])
def test_username_password():
# password not set
mongo_connector = MongoConnector(name='mycon', host='localhost', port=22)
assert mongo_connector._get_mongo_client_kwargs() == {
'host': 'localhost',
'port': 22
}
# password set to None
mongo_connector = MongoConnector(name='mycon',
host='localhost',
port=22,
password=None)
assert mongo_connector._get_mongo_client_kwargs() == {
'host': 'localhost',
'port': 22
}
# password set without username
with pytest.raises(ValueError) as e:
MongoConnector(name='mycon',
host='localhost',
port=22,
password='******')
assert 'username must be set' in str(e.value)
# password and user set
mongo_connector = MongoConnector(name='mycon',
host='localhost',
port=22,
username='******',
password='******')
assert mongo_connector._get_mongo_client_kwargs() == {
'host': 'localhost',
'port': 22,
'username': '******',
'password': '******',
}
def test_client_args_with_ssl(mocker):
"""It should forward parameters to mongo client"""
mongo_client_mock = mocker.patch(
'toucan_connectors.mongo.mongo_connector.pymongo.MongoClient')
connector = MongoConnector(name='my_mongo_con',
host='myhost',
password='******',
username='******',
ssl=True)
connector.client
mongo_client_mock.assert_called_with(host='myhost',
ssl=True,
password='******',
username='******')
def test_get_cache_key(mongo_connector):
"""
It should not raise 'TypeError: Object of type 'MongoClient' is not JSON serializable'
"""
assert mongo_connector.client is not None
assert isinstance(mongo_connector.get_cache_key(), str)
conn1 = MongoConnector(name='aaa',
host='here',
port=42,
username='******',
password='******')
conn2 = MongoConnector(name='aaa',
host='here',
port=42,
username='******',
password='******')
assert conn1.get_cache_key() != conn2.get_cache_key()
def test_client_with_detailed_params():
connector = MongoConnector(name='my_mongo_con', host='myhost', port='123')
assert isinstance(connector.client, pymongo.MongoClient)
def test_get_df(mocker):
class DatabaseMock:
def __init__(self, collection):
self.collections = {collection: pymongo.collection.Collection}
def __getitem__(self, col):
return self.collections[col]
def list_collection_names(self):
return self.collections.keys()
class MongoMock:
def __init__(self, database, collection):
self.data = {database: DatabaseMock(collection)}
def __getitem__(self, row):
return self.data[row]
def close(self):
pass
def list_database_names(self):
return self.data.keys()
snock = mocker.patch('pymongo.MongoClient')
snock.return_value = MongoMock('toucan', 'test_col')
aggregate = mocker.patch('pymongo.collection.Collection.aggregate')
mongo_connector = MongoConnector(name='mycon',
host='localhost',
port=22,
username='******',
password='******')
datasource = MongoDataSource(
name='mycon',
domain='mydomain',
database='toucan',
collection='test_col',
query={'domain': 'domain1'},
)
mongo_connector.get_df(datasource)
datasource = MongoDataSource(
name='mycon',
domain='mydomain',
database='toucan',
collection='test_col',
query=[{
'$match': {
'domain': 'domain1'
}
}],
)
mongo_connector.get_df(datasource)
snock.assert_called_with(host='localhost',
username='******',
password='******',
port=22)
assert snock.call_count == 1 # client is cached
aggregate.assert_called_with([{'$match': {'domain': 'domain1'}}])
assert aggregate.call_count == 2
def test_client_with_mongo_uri():
connector = MongoConnector(name='my_mongo_con',
host='mongodb://*****:*****@myhost:123')
assert isinstance(connector.client, pymongo.MongoClient)