def test_add_select():
"""Test add_select"""
# db init
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3'],
'test4': ['1', '2', '3']
}
data.add_dict(data_dict)
# checks
data2 = copy.deepcopy(data)
data2.add_select([1, 2])
assert data2[0] == data[1]
from dabstract.dataset.select import subsample_by_str
data2 = copy.deepcopy(data)
data2.add_select(subsample_by_str('test3', ['2', '3']))
assert data2[0] == data[1]
from dabstract.dataset.select import random_subsample
data2 = copy.deepcopy(data)
data2.add_select(random_subsample(ratio=0.5))
assert data2['test3'][0] == data2['test4'][0] and data2['test3'][
1] == data2['test4'][1]
data2 = copy.deepcopy(data)
data2.add_select((lambda x, k: x['test3'][k] in ('2', '3')))
assert data2[0] == data[1]
def test_active_keys():
"""Test active_keys"""
# db init
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# checks
data.set_active_keys(['test1', 'test3'])
assert data[0] == {'test1': 1.0, 'test3': '1'}
data.set_active_keys(['test1'])
assert data[0] == 1.0
data.reset_active_keys()
assert data[0] == {
'test1': 1.0,
'test2': 0.0,
'test3': '1',
'test_only': 0.0,
'dataset_id': 0
}
def test_unpack():
"""Test unpack"""
# db init
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# checks
data_unpack = data.unpack(['test1', 'test2', 'test3'])
assert data_unpack[0] == [1.0, 0.0, '1']
def test_keys():
"""Test add_alias"""
# db init
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# checks
assert data.keys() == [
'test1', 'test2', 'test3', 'test_only', 'dataset_id'
]
def test_add_dict():
"""Test add_dict"""
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# checks
assert data[0] == {
'test1': 1.0,
'test_only': 0.0,
'dataset_id': 0,
'test2': 0.0,
'test3': '1'
}
def test_add():
"""Test add"""
data = Dataset()
data.add('test1', np.ones(3))
data.add('test2', np.zeros(3))
data.add('test3', ['1', '2', '3'])
# checks
assert data[0] == {
'test1': 1.0,
'test_only': 0.0,
'dataset_id': 0,
'test2': 0.0,
'test3': '1'
}
def test_add_map():
"""Test add_map"""
# define dataset
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# define processing chain
class custom_processor(Processor):
def process(self, data, **kwargs):
return data + 1, {'multiplier': 3}
class custom_processor2(Processor):
def process(self, data, **kwargs):
return data * kwargs['multiplier'], {}
dp = ProcessingChain()
dp.add(custom_processor)
dp.add(custom_processor2)
# checks
data.add_map('test1', lambda x: x + 1)
assert data[0] == {
'test1': 2.0,
'test2': 0.0,
'test3': '1',
'test_only': 0.0,
'dataset_id': 0,
'dataset_str': 'Dataset'
}
data.add_map('test1', dp)
assert data[0] == {
'test1': 9.0,
'test2': 0.0,
'test3': '1',
'test_only': 0.0,
'dataset_id': 0,
'dataset_str': 'Dataset'
}
def test_add_alias():
"""Test add_alias"""
# db init
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# checks
data.add_alias('test1', 'test1_alias')
assert data['test1'][0] == data['test1_alias'][0]
def test_remove():
"""Test remove"""
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# checks
data.remove('test1')
assert all([
key in ('test2', 'test3', 'test_only', 'dataset_id')
for key in data[0].keys()
])
assert all([
key in ('test2', 'test3', 'test_only', 'dataset_id')
for key in data.keys()
])
def test_concat():
"""Test concat"""
# set dataset 1
data = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data.add_dict(data_dict)
# set dataset 2
data1 = Dataset()
data_dict = {
'test1': np.ones(3),
'test2': np.zeros(3),
'test3': ['1', '2', '3']
}
data1.add_dict(data_dict)
# set dataset 2
data2 = Dataset()
data_dict = {'test2': np.zeros(3), 'test3': ['1', '2', '3']}
data2.add_dict(data_dict)
# checks
data3 = data.concat(data, adjust_base=False)
assert len(data3) == 6
assert data[0] == {
'test1': 1.0,
'test_only': 0.0,
'dataset_id': 0,
'test2': 0.0,
'test3': '1'
}
assert data[-1] == {
'test1': 1.0,
'test_only': 0.0,
'dataset_id': 0,
'test2': 0.0,
'test3': '3'
}
data3 = data.concat(data2, intersect=True, adjust_base=False)
assert len(data3) == 6
assert data3[0] == {
'test2': 0.0,
'test3': '1',
'test_only': 0.0,
'dataset_id': 0
}
assert data3[-1] == {
'test2': 0.0,
'test3': '3',
'test_only': 0.0,
'dataset_id': 1
}
data.concat(data2, intersect=True,
adjust_base=True).concat(data2,
intersect=True,
adjust_base=True)
assert len(data) == 9
assert data[0] == {
'test2': 0.0,
'test3': '1',
'test_only': 0.0,
'dataset_id': 0
}
assert data[-1] == {
'test2': 0.0,
'test3': '3',
'test_only': 0.0,
'dataset_id': 1
}