def test_list_files(self, mock_get_list):
mock_get_list.return_value = [
{
'name': 'a/b/c/file.txt'
},
{
'name': 'a/b/file.txt'
},
{
'name': 'a/file.txt'
},
{
'name': 'b/file.txt'
},
{
'name': 'a/b/c/file2.txt'
},
]
store = ObjectDatastore({})
self.assertEqual([
'a/b/c/file.txt',
'a/b/file.txt',
'a/b/c/file2.txt',
], list(store.list_files('a/b')))
self.assertEqual([
'a/b/c/file.txt',
'a/b/file.txt',
'a/file.txt',
'b/file.txt',
'a/b/c/file2.txt',
], list(store.list_files()))
def test_connect(self, mock_get_connection):
store = ObjectDatastore(self.config)
mock_get_connection.return_value = {"connected": True}
store.connect()
self.assertEqual({'connected': True}, store.connection)
self.assertEqual("(someuser@tenant)", store.user)
def test_read_not_empty(self, mock_container_list):
mock_container_list.return_value = [{"name": "name"}]
config = {
"container": "container",
"file_filter": ".*",
"file_type": None
}
store = ObjectDatastore(config)
data = store.query(None)
self.assertEqual(list(data), [{"name": "name"}])
def test_list_filesizes(self, mock_get_list):
mock_get_list.return_value = [{
'name': 'a/b/c/file.txt',
'bytes': 10
}, {
'name': 'a/b/file.txt',
'bytes': 11
}]
store = ObjectDatastore({})
expected = [('a/b/c/file.txt', 10), ('a/b/file.txt', 11)]
self.assertEqual(list(store.list_filesizes('a/b')), expected)
def test_read_from_xls(self, mock_container_list, mock_object):
mock_container_list.return_value = [{"name": "name"}]
config = {
"container": "container",
"file_filter": ".*",
"file_type": "XLS"
}
store = ObjectDatastore({}, config)
store._read_xls = MagicMock(return_value=[])
data = store.query(None)
self.assertEqual(list(data), [])
store._read_xls.assert_called()
def test_read_from_uva2(self, mock_container_list, mock_object):
mock_container_list.return_value = [{"name": "name"}]
config = {
"container": "container",
"file_filter": ".*",
"file_type": "UVA2"
}
store = ObjectDatastore({}, config)
store._read_uva2 = MagicMock(return_value=[])
data = list(store.query(None))
self.assertEqual(data, [])
store._read_uva2.assert_called()
def test_yield_rows(self):
iterrows = [
('', {
'A': 1,
'B': 2,
'C': 'pandasnull',
'D': 4
}),
('', {
'A': 5,
'B': 6,
'C': 7,
'D': 'pandasnull'
}),
('', {
'A': 'pandasnull',
'B': 'pandasnull',
'C': 'pandasnull',
'D': 'pandasnull'
}),
]
file_info = {
'some': 'file',
'info': 'object',
}
config = {'operators': ['lowercase_keys']}
expected_result = [
{
'a': 1,
'b': 2,
'c': None,
'd': 4,
'_file_info': file_info
},
{
'a': 5,
'b': 6,
'c': 7,
'd': None,
'_file_info': file_info
},
]
store = ObjectDatastore({}, {})
self.assertEqual(expected_result,
list(store._yield_rows(iterrows, file_info, config)))
def test_disconnect(self):
store = ObjectDatastore(self.config)
connection = MagicMock()
store.connection = connection
store.disconnect()
connection.close.assert_called_once()
self.assertIsNone(store.connection)
store.disconnect()
def test_delete_file(self):
store = ObjectDatastore({})
store.connection = MagicMock()
store.container_name = 'container'
store.delete_file('some file')
store.connection.delete_object.assert_called_with(
'container', 'some file')
def test_put_file(self, mock_open, mock_get_connection):
mock_connection = mock_get_connection.return_value
store = ObjectDatastore(self.config,
{'container': 'any container_name'})
store.connect()
mock_file = mock_open.return_value.__enter__.return_value
store.put_file('any src', 'any dest')
mock_connection.put_object.assert_called_with('any container_name',
'any dest',
contents=mock_file)
def _get_cbs_features(path: str) -> dict[str, dict[str, str]]:
"""
Gets the CBS codes from the Objectstore and returns a list of dicts with the naam,
code (wijk or buurt).
:param path: the path to source file
:return: a list of dicts with CBS Code and CBS naam, mapped on the local code.
"""
datastore = ObjectDatastore(
connection_config=get_datastore_config("Basisinformatie"),
read_config={
"file_filter": path,
"file_type": "XLS"
})
datastore.connect()
result = list(datastore.query(''))
datastore.disconnect()
if not result:
raise GOBException(f"No CBS features found for path '{path}'")
return {row[0]: {"code": row[1], "naam": row[2]} for row in result}
def test_init(self):
store = ObjectDatastore(self.config)
self.assertEqual("containerfromenv", store.container_name)
store = ObjectDatastore(self.config, {'container': 'the container'})
self.assertEqual("the container", store.container_name)
def test_missing_config(self):
del self.config['TENANT_NAME']
store = ObjectDatastore(self.config)
with self.assertRaises(GOBException):
store.connect()
def test_generic_exception(self, mock_get_connection):
mock_get_connection.side_effect = Exception
store = ObjectDatastore(self.config)
with self.assertRaises(GOBException):
store.connect()
def test_read(self, mock_container_list):
store = ObjectDatastore(self.config)
data = store.query(None)
self.assertEqual(list(data), [])
mock_container_list.assert_called()
def test_read_xls(self, mock_read, mock_io):
mock_read.return_value = MockUVA2()
store = ObjectDatastore({}, {})
result = [obj for obj in store._read_uva2({}, {}, {})]
self.assertEqual(result, [{"a": 1, "_file_info": {}}])
def test_read_uva2_null_values(self, mock_read, mock_io, mock_isnull):
mock_read.return_value = MockUVA2()
mock_isnull.return_value = True
store = ObjectDatastore({}, {})
result = [obj for obj in store._read_uva2({}, {}, {})]
self.assertEqual(result, [])