def test_unflatten_with_list_issue15():
"""https://github.com/amirziai/flatten/issues/15"""
dic = {
"Required": {
"a": "1",
"b": ["1", "2", "3"],
"c": {
"d": {
"e": [[{
"s1": 1
}, {
"s2": 2
}], [{
"s3": 1
}, {
"s4": 2
}]]
}
},
"f": ["1", "2"]
},
"Optional": {
"x": "1",
"y": ["1", "2", "3"]
}
}
dic_flatten = flatten(dic)
actual = unflatten_list(dic_flatten)
assert actual == dic
def load_all(self, unflatten=True):
d = dict()
for k, v in self.ls():
d[k] = v[:]
if unflatten:
return unflatten_list(d, "/")
else:
return d
def test_flatten_ordered_dict(nested_dict):
fd = flatten_ordered(nested_dict)
assert dict(fd) == {
'a': 1,
'b_c': 3,
'b_d_0': 1,
'b_d_1': 2,
'b_d_2': 3,
'b_e_0_f': 1,
'b_e_1_g': 4
}
assert unflatten_list(fd) == dict(nested_dict)
def batch_iter(self, batch_size=16, **kwargs):
datasets = self.ls()
size = datasets[0][1].shape[0]
n_batches = int(np.ceil(size / batch_size))
for i in range(n_batches):
d = dict()
for k, v in datasets:
if i == n_batches - 1:
# last batch
d[k] = v[(i * batch_size):]
else:
d[k] = v[(i * batch_size):((i + 1) * batch_size)]
yield unflatten_list(d, "/")
def test_unflatten_with_list():
"""Dictionary with lists"""
dic = {
'a': 1,
'b_0': 1,
'b_1': 2,
'c_a': 'a',
'c_b_0': 1,
'c_b_1': 2,
'c_b_2': 3
}
expected = {'a': 1, 'b': [1, 2], 'c': {'a': 'a', 'b': [1, 2, 3]}}
actual = unflatten_list(dic)
assert actual == expected
dic = {'a': 1, 'b_0': 5}
expected = {'a': 1, 'b': [5]}
actual = unflatten_list(dic)
assert actual == expected
dic = {'a': 1, 'b:0': 5}
expected = {'a': 1, 'b': [5]}
actual = unflatten_list(dic, ':')
assert actual == expected
def load_all(self, unflatten=True):
"""Load the whole file
# Arguments
unflatten: if True, nest/unflatten the keys.
e.g. an entry `f['/foo/bar']` would need to be accessed
using two nested `get` call: `f['foo']['bar']`
"""
d = dict()
for k, v in self.ls():
d[k] = v[:]
if unflatten:
return unflatten_list(d, "/")
else:
return d
def batch_iter(self, batch_size=16, **kwargs):
"""Create a batch iterator over the whole file
# Arguments
batch_size: batch size
**kwargs: ignored argument. Used for consistency with other dataloaders
"""
datasets = self.ls()
n_batches = int(np.ceil(len(self) / batch_size))
for i in range(n_batches):
d = dict()
for k, v in datasets:
if i == n_batches - 1:
# last batch
d[k] = v[(i * batch_size):]
else:
d[k] = v[(i * batch_size):((i + 1) * batch_size)]
yield unflatten_list(d, "/")
def test_unflatten_with_list_deep():
dic = {
'a': [{
'b': [{
'c': [{
'a': 5,
'b': {
'a': [1, 2, 3]
},
'c': {
'x': 3
}
}]
}]
}]
}
dic_flatten = flatten(dic)
actual = unflatten_list(dic_flatten)
assert actual == dic
def test_unflatten_with_list_custom_separator():
"""Complex dictionary with lists"""
dic = {
'a:b': 'str0',
'c:0:d:0:e': 'str1',
'c:1:d:0:e': 'str4',
'c:1:f': 'str5',
'c:0:f': 'str2',
'c:1:g': 'str6',
'c:0:g': 'str3',
'h:d:0:e': 'str7',
'h:i:0:f': 'str8',
'h:i:0:g': 'str9'
}
expected = {
'a': {
'b': 'str0'
},
'c': [{
'd': [{
'e': 'str1'
}],
'f': 'str2',
'g': 'str3'
}, {
'd': [{
'e': 'str4'
}],
'f': 'str5',
'g': 'str6'
}],
'h': {
'd': [{
'e': 'str7'
}],
'i': [{
'f': 'str8',
'g': 'str9'
}]
}
}
actual = unflatten_list(dic, ':')
assert actual == expected
def test_unflatten_with_list_nested():
dic = {"a": [[{"b": 1}], [{"d": 1}]]}
dic_flatten = flatten(dic)
actual = unflatten_list(dic_flatten)
assert actual == dic
