def _back_to_walk_path(parent, key, data, data_type, **_):
if parent is None and key is None:
return
work_list.append(
dict(data=data, parent=parent, key=key,
path=path.add((c.KEY_LITERAL | guess_type(parent),
key, path_pos))))
def test_guess_type(self):
tests = [[tuple(), c.TYPE_LIST], [list(), c.TYPE_LIST],
[dict(), c.TYPE_DICT], ['', c.TYPE_LEAF], [None, c.TYPE_LEAF],
[6, c.TYPE_LEAF]]
for obj, obj_type in tests:
self.assertEqual(
guess_type(obj), obj_type,
"We can correctly get the type for '%s'" % str(obj))
def test_guess_type(self):
tests = [
[tuple(), c.TYPE_LIST],
[list(), c.TYPE_LIST],
[dict(), c.TYPE_DICT],
['', c.TYPE_LEAF],
[None, c.TYPE_LEAF],
[6, c.TYPE_LEAF]
]
for obj, obj_type in tests:
self.assertEqual(
guess_type(obj), obj_type,
"We can correctly get the type for '%s'" % str(obj))
def walk(data, function, root=None, parent=None, key=None, path=None):
if root is None:
root = data
if path is None:
path = BranchingList()
data_type = guess_type(data)
instruction = function(data=data,
data_type=data_type,
parent=parent,
key=key,
path=path)
if instruction != c.WALK_CONTINUE:
return instruction
if data_type == c.TYPE_LEAF:
return
else:
if data_type == c.TYPE_DICT:
items = data.iteritems()
elif data_type == c.TYPE_LIST:
items = enumerate(data)
for key, item in items:
instruction = walk(
# Things that don't change
function=function,
root=root,
# Things that do
data=item,
key=key,
parent=data,
path=path.add((c.KEY_LITERAL | data_type, key)))
if instruction == c.WALK_PRUNE:
break
if instruction == c.WALK_TERMINATE:
return c.WALK_TERMINATE
def walk(data, function, root=None, parent=None, key=None, path=None):
if root is None:
root = data
if path is None:
path = BranchingList()
data_type = guess_type(data)
instruction = function(
data=data, data_type=data_type, parent=parent, key=key, path=path)
if instruction != c.WALK_CONTINUE:
return instruction
if data_type == c.TYPE_LEAF:
return
elif data_type == c.TYPE_DICT:
items = data.iteritems()
elif data_type == c.TYPE_LIST:
items = enumerate(data)
# Prevent changing size:
items = tuple(items)
for key, item in items:
instruction = walk(
# Things that don't change
function=function, root=root,
# Things that do
data=item, key=key, parent=data,
path=path.add((c.KEY_LITERAL | data_type, key))
)
if instruction == c.WALK_PRUNE:
break
if instruction == c.WALK_TERMINATE:
return c.WALK_TERMINATE
def _walk_path(context, data, path_pos, parent, key, path):
data_type = guess_type(data)
instruction = context['function'](
# Things which change all the time
data=data, data_type=data_type, path_pos=path_pos,
parent=parent, key=key, path=path,
# Things we calculate to be nice
terminal=path_pos == len(context['path_parts']),
**context)
if instruction != c.WALK_CONTINUE:
return instruction
# We've run out of path
if path_pos >= len(context['path_parts']):
return
key_type, key = context['path_parts'][path_pos]
# Type mismatch!
if not key_type & data_type:
if context['on_mismatch'] == c.ON_MISMATCH_FAIL:
raise ValueError('Expected %s but found %s at %s: %s' % (
c.STRINGS[key_type & c.TYPE_MASK],
c.STRINGS[data_type],
data,
canonical_path(context['path_parts'])
))
elif context['on_mismatch'] == c.ON_MISMATCH_CONTINUE:
return c.WALK_CONTINUE
else:
raise Exception('wut?')
# It's a super mad recursion into the data structure
elif key_type & c.KEY_RECURSE:
recurse_context = dict(context, on_mismatch=c.ON_MISMATCH_CONTINUE)
if key_type & c.KEY_WILD:
return walk(
data=data,
function=lambda **kwargs: _walk_path(
context=recurse_context, data=kwargs['data'],
path_pos=path_pos + 1, parent=kwargs['parent'],
key=None, path=kwargs['path']),
parent=parent, key=None, path=path)
elif key_type & c.KEY_LITERAL:
return walk(
data=data,
function=lambda **kwargs: _walk_path(
context=recurse_context, data=kwargs['data'],
path_pos=path_pos + 1, parent=kwargs['parent'],
key=None, path=kwargs['path']) if kwargs['key'] == key
else c.WALK_CONTINUE,
parent=parent, key=None, path=path)
else:
raise ValueError('HOW DO YOU EVEN')
# A literal, we know exactly where to go
elif key_type & c.KEY_LITERAL:
missing = (key_type & c.TYPE_DICT and key not in data) or (
key_type & c.TYPE_LIST and key >= len(data))
if missing:
if context['on_missing'] == c.ON_MISSING_CONTINUE:
return
elif context['on_missing'] == c.ON_MISSING_CREATE:
data = _auto_fill(data, data_type, key, context['path_parts'],
path_pos)
else:
raise Exception('wut?')
return _walk_path(context, data=data[key], key=key, parent=data,
path=path.add((c.KEY_LITERAL | data_type, key)),
path_pos=path_pos + 1)
# A wild card, we go everywhere
elif key_type & c.KEY_WILD:
if data_type & c.TYPE_LIST:
keys = xrange(len(data))
elif data_type & c.TYPE_DICT:
keys = data.iterkeys()
else:
raise ValueError('Unknown sub-object type')
for key in keys:
instruction = _walk_path(
context, data=data[key], key=key,
parent=data, path_pos=path_pos + 1,
path=path.add((c.KEY_LITERAL | data_type, key))
)
if instruction == c.WALK_PRUNE:
break
if instruction == c.WALK_TERMINATE:
return c.WALK_TERMINATE
else:
raise Exception('Bad key type')
def _walk_path(context, data, path_pos, parent, key, path):
data_type = guess_type(data)
instruction = context['function'](
# Things which change all the time
data=data, data_type=data_type, path_pos=path_pos,
parent=parent, key=key, path=path,
# Things we calculate to be nice
terminal=path_pos == len(context['path_parts']),
**context)
if instruction != c.WALK_CONTINUE:
return instruction
# We've run out of path
if path_pos >= len(context['path_parts']):
return
key_type, key = context['path_parts'][path_pos]
if key_type & c.TRAVERSAL_RECURSE:
return _path_recursion(
data=data, parent=parent, path=path, path_pos=path_pos,
context=context)
elif not key_type & data_type:
if context['on_mismatch'] == c.ON_MISMATCH_FAIL:
raise ValueError('Expected %s but found %s at %s: %s' % (
c.STRINGS[key_type & c.TYPE_MASK],
c.STRINGS[data_type],
data,
compact_path(context['path_parts'])
))
elif context['on_mismatch'] == c.ON_MISMATCH_CONTINUE:
return c.WALK_CONTINUE
else:
raise Exception('wut?')
# It's a super mad recursion into the data structure
# A literal, we know exactly where to go
elif key_type & c.KEY_LITERAL:
try:
data[key]
except (KeyError, IndexError):
if context['on_missing'] == c.ON_MISSING_CONTINUE:
return
elif context['on_missing'] == c.ON_MISSING_CREATE:
data = _auto_fill(data, data_type, key,
context['path_parts'], path_pos)
return _walk_path(
context, data=data[key], key=key, parent=data,
path=path.add((c.KEY_LITERAL | data_type, key, path_pos)),
path_pos=path_pos + 1)
elif key_type & (c.KEY_WILD | c.KEY_SLICE):
# A wild key
if key_type & c.KEY_WILD:
if data_type & c.TYPE_LIST:
keys = xrange(len(data))
elif data_type & c.TYPE_DICT:
keys = data.iterkeys()
else:
raise ValueError('Unknown sub-object type')
# Then it's a slice
else:
if data_type & c.TYPE_LIST and isinstance(key, slice):
stop = max(key.stop or 0, len(data))
keys = xrange(*key.indices(stop))
else:
# Literal values
keys = key
for key in keys:
# TODO! - Some copy paste here from above.
try:
data[key]
except (KeyError, IndexError):
if context['on_missing'] == c.ON_MISSING_CONTINUE:
continue
elif context['on_missing'] == c.ON_MISSING_CREATE:
data = _auto_fill(data, data_type, key,
context['path_parts'], path_pos)
else:
raise
instruction = _walk_path(
context, data=data[key], key=key,
parent=data, path_pos=path_pos + 1,
path=path.add((c.KEY_LITERAL | data_type, key, path_pos)))
if instruction == c.WALK_PRUNE:
break
if instruction == c.WALK_TERMINATE:
return c.WALK_TERMINATE
else:
raise Exception('Bad key type')
def _walk_path(context, data, path_pos, parent, key, path):
data_type = guess_type(data)
instruction = context['function'](
# Things which change all the time
data=data,
data_type=data_type,
path_pos=path_pos,
parent=parent,
key=key,
path=path,
# Things we calculate to be nice
terminal=path_pos == len(context['path_parts']),
**context)
if instruction != c.WALK_CONTINUE:
return instruction
# We've run out of path
if path_pos >= len(context['path_parts']):
return
key_type, key = context['path_parts'][path_pos]
# Type mismatch!
if not key_type & data_type:
if context['on_mismatch'] == c.ON_MISMATCH_FAIL:
raise ValueError(
'Expected %s but found %s at %s: %s' %
(c.STRINGS[key_type & c.TYPE_MASK], c.STRINGS[data_type], data,
canonical_path(context['path_parts'])))
elif context['on_mismatch'] == c.ON_MISMATCH_CONTINUE:
return c.WALK_CONTINUE
else:
raise Exception('wut?')
# It's a super mad recursion into the data structure
elif key_type & c.KEY_RECURSE:
recurse_context = dict(context, on_mismatch=c.ON_MISMATCH_CONTINUE)
if key_type & c.KEY_WILD:
return walk(
data=data,
function=lambda **kwargs: _walk_path(context=recurse_context,
data=kwargs['data'],
path_pos=path_pos + 1,
parent=kwargs['parent'],
key=None,
path=kwargs['path']),
parent=parent,
key=None,
path=path)
elif key_type & c.KEY_LITERAL:
return walk(
data=data,
function=lambda **kwargs: _walk_path(context=recurse_context,
data=kwargs['data'],
path_pos=path_pos + 1,
parent=kwargs['parent'],
key=None,
path=kwargs['path'])
if kwargs['key'] == key else c.WALK_CONTINUE,
parent=parent,
key=None,
path=path)
else:
raise ValueError('HOW DO YOU EVEN')
# A literal, we know exactly where to go
elif key_type & c.KEY_LITERAL:
missing = (key_type & c.TYPE_DICT
and key not in data) or (key_type & c.TYPE_LIST
and key >= len(data))
if missing:
if context['on_missing'] == c.ON_MISSING_CONTINUE:
return
elif context['on_missing'] == c.ON_MISSING_CREATE:
data = _auto_fill(data, data_type, key, context['path_parts'],
path_pos)
else:
raise Exception('wut?')
return _walk_path(context,
data=data[key],
key=key,
parent=data,
path=path.add((c.KEY_LITERAL | data_type, key)),
path_pos=path_pos + 1)
# A wild card, we go everywhere
elif key_type & c.KEY_WILD:
if data_type & c.TYPE_LIST:
keys = xrange(len(data))
elif data_type & c.TYPE_DICT:
keys = data.iterkeys()
else:
raise ValueError('Unknown sub-object type')
for key in keys:
instruction = _walk_path(context,
data=data[key],
key=key,
parent=data,
path_pos=path_pos + 1,
path=path.add(
(c.KEY_LITERAL | data_type, key)))
if instruction == c.WALK_PRUNE:
break
if instruction == c.WALK_TERMINATE:
return c.WALK_TERMINATE
else:
raise Exception('Bad key type')