def sharding(sorted_iterable, size, accuracy=None, joiner=''.join):
if accuracy is None:
accuracy = size / 10
t = make_trie(sorted_iterable, node_max_num=accuracy)
n = 0
prev_key = None
rst = []
# iterate root node.
t = {'': t}
for ks, node in dictutil.depth_iter(
t, is_allowed=lambda ks, v: v.is_eol or len(v) == 0):
if n >= size:
rst.append((prev_key, n))
# strip empty root path
prev_key = ks[1:]
prev_key = joiner(prev_key)
n = 0
if len(node) == 0:
n += node.n
else:
# node.is_eol == True
n += 1
rst.append((prev_key, n))
return rst
def add_command_help(commands):
new_cmds = copy.deepcopy(commands)
help_msgs = new_cmds.get('__add_help__')
desc = new_cmds.get('__description__')
for k in ('__add_help__', '__description__'):
if new_cmds.has_key(k):
del new_cmds[k]
if help_msgs is None:
return new_cmds, None
parser = argparse.ArgumentParser(description=desc, epilog='\n')
subparsers = parser.add_subparsers(help=' command(s) to select ...')
for cmds, execute_able in dictutil.depth_iter(new_cmds):
help = help_msgs.get(tuple(cmds), '')
cmd = ' '.join(cmds)
cmd_parser = subparsers.add_parser(cmd, help=help)
if need_param_help(execute_able):
call_able = execute_able[0]
param_msgs = execute_able[1:]
params = add_param_help(cmd_parser, param_msgs)
# delete help message
dictutil.make_setter(cmds)(new_cmds, (call_able, params))
return new_cmds, parser
def test_depth_iter_ks(self):
idx = 0
ks = ['mykey']
_in = {'k1': {'k11': 'v11'}, 'k2': 'v2'}
_out = [
(['mykey', 'k1', 'k11'], 'v11'),
(['mykey', 'k2'], 'v2'),
]
_mes = 'test argument ks in dictutil.dict_depth_iter()'
for rst in dictutil.depth_iter(_in, ks=ks):
self.assertEqual(
_out[idx],
rst,
('input: {_in}, output: {rst}, expected: {_out},'
'message: {_mes}').format(
_in=repr(_in),
_out=repr(rst),
rst=repr(_out[idx]),
_mes=_mes
)
)
idx = idx + 1
def test_trie_prefix_count(self):
t = strutil.make_trie(self.strs, node_max_num=3)
dd('==== trie:')
dd(str(t))
# all prefixes, might be also a whole string.
for ks, v in dictutil.depth_iter(t, intermediate=True):
dd('==== got keys:', ks, 'sub trie:')
dd(v)
prefix = ''.join(ks)
expected = len([x for x in self.strs if x.startswith(prefix)])
key_path = '.'.join(ks)
subtrie = dictutil.get(t, key_path, default=-1)
rst = subtrie.n
dd('prefix:', prefix, 'trie key_path:', key_path, 'sub trie:')
dd(str(subtrie))
dd('expected:', expected)
dd('rst:', rst)
self.assertEqual(expected, rst)
# Leaf node number limit
if len(v) == 0:
self.assertLessEqual(rst, 3)
def test_depth_iter_maxdepth(self):
idx = 0
_in = {
'k1': {
'k11': {
'k111': {
'k1111': 'v1111'
},
'empty': {},
},
},
}
_out = [
(['k1'], {'k11': {'k111': {'k1111': 'v1111'}, 'empty': {}}}),
(['k1', 'k11'], {'k111': {'k1111': 'v1111'}, 'empty': {}}),
(['k1', 'k11', 'empty'], {}),
(['k1', 'k11', 'k111'], {'k1111': 'v1111'}),
(['k1', 'k11', 'k111', 'k1111'], 'v1111')
]
for depth in range(1, 5):
for rst in dictutil.depth_iter(_in, maxdepth=depth):
dd('depth:', depth)
dd('_in:', _in)
dd('_out[idx]:', _out[idx])
dd('rst:', rst)
self.assertEqual(_out[idx], rst)
idx = idx + 1
def test_depth_iter_empty_as_leaf_and_intermediate(self):
_in = {
'k1': {
'k11': {
'k111': {
'k1111': 'v1111'
},
'empty': {},
},
},
}
_out = [
(['k1'], {'k11': {'k111': {'k1111': 'v1111'}, 'empty': {}}}),
(['k1', 'k11'], {'k111': {'k1111': 'v1111'}, 'empty': {}}),
(['k1', 'k11', 'empty'], {}),
(['k1', 'k11', 'k111'], {'k1111': 'v1111'}),
(['k1', 'k11', 'k111', 'k1111'], 'v1111')
]
idx = 0
for rst in dictutil.depth_iter(_in, intermediate=True, empty_leaf=True):
self.assertEqual(_out[idx], rst)
idx = idx + 1
self.assertEqual(len(_out), idx)
def sharding(sorted_iterable, size, accuracy=None, joiner=''.join):
if accuracy is None:
accuracy = size / 10
t = make_trie(sorted_iterable, node_max_num=accuracy)
n = 0
prev_key = None
rst = []
# iterate root node.
t = {'': t}
for ks, node in dictutil.depth_iter(t, is_allowed=lambda ks, v: v.is_eol or len(v) == 0):
if n >= size:
rst.append((prev_key, n))
# strip empty root path
prev_key = ks[1:]
prev_key = joiner(prev_key)
n = 0
if len(node) == 0:
n += node.n
else:
# node.is_eol == True
n += 1
rst.append((prev_key, n))
return rst
def test_depth_iter_is_allowed(self):
_in = {
'k1': {
'k11': {
'k111': {
'k1111': 'v1111'
},
'empty': {},
},
},
}
cases = (
({'is_allowed': lambda ks, v: isinstance(v, dict) and len(v) == 0,
},
['k1', 'k11', 'empty'], {},
'choose only empty dict value',
),
({'is_allowed': lambda ks, v: isinstance(v, str),
},
['k1', 'k11', 'k111', 'k1111'], 'v1111',
'choose only string value',
),
({'is_allowed': lambda ks, v: isinstance(v, str),
'intermediate': True,
},
['k1', 'k11', 'k111', 'k1111'], 'v1111',
'is_allowed should override intermediate',
),
({'is_allowed': lambda ks, v: isinstance(v, str),
'empty_leaf': True,
},
['k1', 'k11', 'k111', 'k1111'], 'v1111',
'is_allowed should override empty_leaf',
),
)
for kwargs, expected_ks, expected_v, msg in cases:
dd(msg)
dd(kwargs)
dd('expected:', expected_ks, expected_v)
for ks, v in dictutil.depth_iter(_in, **kwargs):
self.assertEqual(expected_ks, ks)
self.assertEqual(expected_v, v)
def test_trie_iterable(self):
t = strutil.make_trie(self.iterables, node_max_num=3)
dd('trie:')
dd(str(t))
self.assertEqual(len(self.iterables), t.n)
for ks, v in dictutil.depth_iter(t, is_allowed=lambda ks, v: v.is_eol):
s = tuple(ks)
dd('whole string: ', s, v)
self.assertTrue(s in self.iterables)
def test_trie_whole_string(self):
t = strutil.make_trie(self.strs, node_max_num=3)
dd('trie:')
dd(str(t))
self.assertEqual(len(self.strs), t.n)
# only whole strings
for ks, v in dictutil.depth_iter(t, is_allowed=lambda ks, v: v.is_eol):
s = ''.join(ks)
dd('whole string: ', s, v)
self.assertTrue(s in self.strs)
def gen_keys(data):
_rst = []
for keys, value in dictutil.depth_iter(data, maxdepth=depth):
if len(keys) == 1:
item_params = keys[0]
else:
item_params = ','.join(keys)
item_value = value
m = "{0} {1}[{2}] {3}".format(
'-', self.item_key, item_params, item_value)
_rst.append(m)
return _rst
def test_node_max_num(self):
t = strutil.make_trie(self.strs, node_max_num=1)
for ks, v in dictutil.depth_iter(t, empty_leaf=True):
key_path = '.'.join(ks)
subtrie = dictutil.get(t, key_path, default=-1)
self.assertEqual(1, subtrie.n)
# all in one node
t = strutil.make_trie(self.strs, node_max_num=10000)
dd()
dd(str(t))
self.assertEqual(len(self.strs), t.n)
self.assertEqual(0, len(t))
def test_depth_iter_default(self):
cases = (
({}, []),
({'k1': 'v1'},
[(['k1'], 'v1')]
),
({'k1': 'v1', 'k2': 'v2'},
[
(['k1'], 'v1'),
(['k2'], 'v2'),
]),
({'k1': {'k11': 'v11'},
'k2': 'v2',
'empty': {},
},
([
(['k1', 'k11'], 'v11'),
(['k2'], 'v2'),
]),
)
)
for _in, _out in cases:
idx = 0
for rst in dictutil.depth_iter(_in):
self.assertEqual(
_out[idx],
rst,
('input: {_in}, output: {rst}, expected: {_out}').format(
_in=repr(_in),
_out=repr(rst),
rst=repr(_out[idx])
)
)
idx = idx + 1