def _merge_item(d, key, value, overwrite=True):
if key in d:
item = d.get(key, None)
if type_util.is_dict(item) and type_util.is_dict(value):
_merge_dict(item, value, overwrite)
elif overwrite:
d[key] = value
else:
d[key] = value
def _merge_item(d, key, value, overwrite=True, concat=False):
if key in d:
item = d.get(key, None)
if type_util.is_dict(item) and type_util.is_dict(value):
_merge_dict(item, value, overwrite=overwrite, concat=concat)
elif concat and type_util.is_list(item) and type_util.is_list(value):
item += value
elif overwrite:
d[key] = value
else:
d[key] = value
def _getitem_by_keys(self, keys):
parent, key, _ = keylist_util.get_item(self, keys)
if type_util.is_dict(parent):
return parent[key]
elif type_util.is_list(parent):
return parent[key]
raise KeyError('Invalid keys: "{}"'.format(keys))
def _getitem_by_keys(self, keys):
parent, key, _ = keylist_util.get_item(self, keys)
if type_util.is_dict(parent):
return parent[key]
elif type_util.is_list(parent):
return parent[key]
raise KeyError
def _get_by_keys(self, keys, default=None):
parent, key, _ = keylist_util.get_item(self, keys)
if type_util.is_dict(parent):
return parent.get(key, default)
elif type_util.is_list(parent):
return parent[key]
return default
def _contains_by_keys(self, keys):
parent, _, _ = keylist_util.get_item(self, keys)
if type_util.is_dict(parent):
return True
elif type_util.is_list(parent):
return True
return False
def _get_keylist(item, parent_keys):
keylist = []
for key, value in item.items():
keys = parent_keys + [key]
keylist += [keys]
if type_util.is_dict(value):
keylist += _get_keylist(value, keys)
return keylist
def _get_item_key_and_value(item, key):
if type_util.is_list_or_tuple(item):
index = _get_index(key)
if index is not None:
return (index, item[index])
elif type_util.is_dict(item):
return (key, item[key])
raise KeyError('Invalid key: "{}"'.format(key))
def _pop_by_keys(self, keys, *args):
parent, key, _ = keylist_util.get_item(self, keys)
if type_util.is_dict(parent):
return parent.pop(key, *args)
elif type_util.is_list(parent):
return parent.pop(key)
if args:
return args[0]
raise KeyError
def _parse_dict(val):
serializer = JSONSerializer()
try:
d = serializer.decode(val)
if type_util.is_dict(d):
return d
return None
except Exception:
return None
def _unflatten_item(key, value, separator):
keys = key.split(separator)
if type_util.is_dict(value):
return (
keys,
unflatten(value, separator=separator),
)
return (
keys,
value,
)
def test_encode_decode_pickle(self):
d = {
'date': dt.datetime(year=1985, month=4, day=3),
}
serializer = PickleSerializer()
s = serializer.encode(d)
print(s)
self.assertTrue(type_util.is_string(s))
r = serializer.decode(s)
self.assertTrue(type_util.is_dict(d))
self.assertEqual(d, r)
def _pop_by_keys(self, keys, *args):
parent, key, _ = keylist_util.get_item(self, keys)
if type_util.is_dict(parent):
return parent.pop(key, *args)
elif type_util.is_list(parent):
return parent.pop(key)
elif type_util.is_tuple(parent):
# raise the standard TypeError
del parent[key]
if args:
return args[0]
raise KeyError('Invalid keys: "{}"'.format(keys))
def groupby(items, key):
if not type_util.is_list(items):
raise ValueError('items should be a list of dicts.')
items_grouped = {}
for item in items:
if not type_util.is_dict(item):
raise ValueError('item should be a dict.')
group = item.get(key)
if group not in items_grouped:
items_grouped[group] = []
items_grouped[group].append(item.copy())
return items_grouped
def check_keys(d, separator):
"""
Check if dict keys contain keypath separator.
"""
if not type_util.is_dict(d) or not separator:
return
def check_key(parent, key, value):
if key and type_util.is_string(key) and separator in key:
raise ValueError(
'keys should not contain keypath separator '
'\'{}\', found: \'{}\'.'.format(separator, key))
traverse(d, check_key)
def encode(self, d, **kwargs):
parser = ConfigParser(**kwargs)
for key, value in d.items():
if not type_util.is_dict(value):
parser.set(default_section, key, '{}'.format(value))
continue
section = key
parser.add_section(section)
for option_key, option_value in value.items():
parser.set(section, option_key, '{}'.format(option_value))
str_data = StringIO()
parser.write(str_data)
return str_data.getvalue()
def _get_item_key_and_value(item, key):
if type_util.is_list(item):
index = _get_index(key)
if index is not None:
return (
index,
item[index],
)
elif type_util.is_dict(item):
return (
key,
item[key],
)
raise KeyError
def _decode(s, format, **kwargs):
try:
content = io_util.read_content(s)
# decode content using the given format
data = io_util.decode(content, format, **kwargs)
if type_util.is_dict(data):
return data
elif type_util.is_list(data):
# force list to dict
return {'values': data}
else:
raise ValueError(
'Invalid data type: {}, expected dict or list.'.format(
type(data)))
except Exception as e:
raise ValueError(
'Invalid data or url or filepath argument: {}\n{}'.format(
s, e))
def _flatten_item(d, base_dict, base_key, separator):
new_dict = base_dict
keys = list(d.keys())
for key in keys:
new_key = _flatten_key(base_key, key, separator)
value = d.get(key, None)
if type_util.is_dict(value):
new_value = _flatten_item(value,
base_dict=new_dict,
base_key=new_key,
separator=separator)
new_dict.update(new_value)
continue
if new_key in new_dict:
raise KeyError(
'Invalid key: "{}", key already in flatten dict.'.format(
new_key))
new_dict[new_key] = value
return new_dict
def _merge_item(d, key, value):
item = d.get(key, None)
if type_util.is_dict(item) and type_util.is_dict(value):
_merge_dict(item, value)
else:
d[key] = value
def _get_keylist_for_value(value, parent_keys, indexes):
if type_util.is_dict(value):
return _get_keylist_for_dict(value, parent_keys, indexes)
elif type_util.is_list(value) and indexes:
return _get_keylist_for_list(value, parent_keys, indexes)
return []
