def _back_to_walk_path(parent, key, data, data_type, **_): if parent is None and key is None: return work_list.append( dict(data=data, parent=parent, key=key, path=path.add((c.KEY_LITERAL | guess_type(parent), key, path_pos))))
def test_guess_type(self): tests = [[tuple(), c.TYPE_LIST], [list(), c.TYPE_LIST], [dict(), c.TYPE_DICT], ['', c.TYPE_LEAF], [None, c.TYPE_LEAF], [6, c.TYPE_LEAF]] for obj, obj_type in tests: self.assertEqual( guess_type(obj), obj_type, "We can correctly get the type for '%s'" % str(obj))
def test_guess_type(self): tests = [ [tuple(), c.TYPE_LIST], [list(), c.TYPE_LIST], [dict(), c.TYPE_DICT], ['', c.TYPE_LEAF], [None, c.TYPE_LEAF], [6, c.TYPE_LEAF] ] for obj, obj_type in tests: self.assertEqual( guess_type(obj), obj_type, "We can correctly get the type for '%s'" % str(obj))
def walk(data, function, root=None, parent=None, key=None, path=None): if root is None: root = data if path is None: path = BranchingList() data_type = guess_type(data) instruction = function(data=data, data_type=data_type, parent=parent, key=key, path=path) if instruction != c.WALK_CONTINUE: return instruction if data_type == c.TYPE_LEAF: return else: if data_type == c.TYPE_DICT: items = data.iteritems() elif data_type == c.TYPE_LIST: items = enumerate(data) for key, item in items: instruction = walk( # Things that don't change function=function, root=root, # Things that do data=item, key=key, parent=data, path=path.add((c.KEY_LITERAL | data_type, key))) if instruction == c.WALK_PRUNE: break if instruction == c.WALK_TERMINATE: return c.WALK_TERMINATE
def walk(data, function, root=None, parent=None, key=None, path=None): if root is None: root = data if path is None: path = BranchingList() data_type = guess_type(data) instruction = function( data=data, data_type=data_type, parent=parent, key=key, path=path) if instruction != c.WALK_CONTINUE: return instruction if data_type == c.TYPE_LEAF: return elif data_type == c.TYPE_DICT: items = data.iteritems() elif data_type == c.TYPE_LIST: items = enumerate(data) # Prevent changing size: items = tuple(items) for key, item in items: instruction = walk( # Things that don't change function=function, root=root, # Things that do data=item, key=key, parent=data, path=path.add((c.KEY_LITERAL | data_type, key)) ) if instruction == c.WALK_PRUNE: break if instruction == c.WALK_TERMINATE: return c.WALK_TERMINATE
def _walk_path(context, data, path_pos, parent, key, path): data_type = guess_type(data) instruction = context['function']( # Things which change all the time data=data, data_type=data_type, path_pos=path_pos, parent=parent, key=key, path=path, # Things we calculate to be nice terminal=path_pos == len(context['path_parts']), **context) if instruction != c.WALK_CONTINUE: return instruction # We've run out of path if path_pos >= len(context['path_parts']): return key_type, key = context['path_parts'][path_pos] # Type mismatch! if not key_type & data_type: if context['on_mismatch'] == c.ON_MISMATCH_FAIL: raise ValueError('Expected %s but found %s at %s: %s' % ( c.STRINGS[key_type & c.TYPE_MASK], c.STRINGS[data_type], data, canonical_path(context['path_parts']) )) elif context['on_mismatch'] == c.ON_MISMATCH_CONTINUE: return c.WALK_CONTINUE else: raise Exception('wut?') # It's a super mad recursion into the data structure elif key_type & c.KEY_RECURSE: recurse_context = dict(context, on_mismatch=c.ON_MISMATCH_CONTINUE) if key_type & c.KEY_WILD: return walk( data=data, function=lambda **kwargs: _walk_path( context=recurse_context, data=kwargs['data'], path_pos=path_pos + 1, parent=kwargs['parent'], key=None, path=kwargs['path']), parent=parent, key=None, path=path) elif key_type & c.KEY_LITERAL: return walk( data=data, function=lambda **kwargs: _walk_path( context=recurse_context, data=kwargs['data'], path_pos=path_pos + 1, parent=kwargs['parent'], key=None, path=kwargs['path']) if kwargs['key'] == key else c.WALK_CONTINUE, parent=parent, key=None, path=path) else: raise ValueError('HOW DO YOU EVEN') # A literal, we know exactly where to go elif key_type & c.KEY_LITERAL: missing = (key_type & c.TYPE_DICT and key not in data) or ( key_type & c.TYPE_LIST and key >= len(data)) if missing: if context['on_missing'] == c.ON_MISSING_CONTINUE: return elif context['on_missing'] == c.ON_MISSING_CREATE: data = _auto_fill(data, data_type, key, context['path_parts'], path_pos) else: raise Exception('wut?') return _walk_path(context, data=data[key], key=key, parent=data, path=path.add((c.KEY_LITERAL | data_type, key)), path_pos=path_pos + 1) # A wild card, we go everywhere elif key_type & c.KEY_WILD: if data_type & c.TYPE_LIST: keys = xrange(len(data)) elif data_type & c.TYPE_DICT: keys = data.iterkeys() else: raise ValueError('Unknown sub-object type') for key in keys: instruction = _walk_path( context, data=data[key], key=key, parent=data, path_pos=path_pos + 1, path=path.add((c.KEY_LITERAL | data_type, key)) ) if instruction == c.WALK_PRUNE: break if instruction == c.WALK_TERMINATE: return c.WALK_TERMINATE else: raise Exception('Bad key type')
def _walk_path(context, data, path_pos, parent, key, path): data_type = guess_type(data) instruction = context['function']( # Things which change all the time data=data, data_type=data_type, path_pos=path_pos, parent=parent, key=key, path=path, # Things we calculate to be nice terminal=path_pos == len(context['path_parts']), **context) if instruction != c.WALK_CONTINUE: return instruction # We've run out of path if path_pos >= len(context['path_parts']): return key_type, key = context['path_parts'][path_pos] if key_type & c.TRAVERSAL_RECURSE: return _path_recursion( data=data, parent=parent, path=path, path_pos=path_pos, context=context) elif not key_type & data_type: if context['on_mismatch'] == c.ON_MISMATCH_FAIL: raise ValueError('Expected %s but found %s at %s: %s' % ( c.STRINGS[key_type & c.TYPE_MASK], c.STRINGS[data_type], data, compact_path(context['path_parts']) )) elif context['on_mismatch'] == c.ON_MISMATCH_CONTINUE: return c.WALK_CONTINUE else: raise Exception('wut?') # It's a super mad recursion into the data structure # A literal, we know exactly where to go elif key_type & c.KEY_LITERAL: try: data[key] except (KeyError, IndexError): if context['on_missing'] == c.ON_MISSING_CONTINUE: return elif context['on_missing'] == c.ON_MISSING_CREATE: data = _auto_fill(data, data_type, key, context['path_parts'], path_pos) return _walk_path( context, data=data[key], key=key, parent=data, path=path.add((c.KEY_LITERAL | data_type, key, path_pos)), path_pos=path_pos + 1) elif key_type & (c.KEY_WILD | c.KEY_SLICE): # A wild key if key_type & c.KEY_WILD: if data_type & c.TYPE_LIST: keys = xrange(len(data)) elif data_type & c.TYPE_DICT: keys = data.iterkeys() else: raise ValueError('Unknown sub-object type') # Then it's a slice else: if data_type & c.TYPE_LIST and isinstance(key, slice): stop = max(key.stop or 0, len(data)) keys = xrange(*key.indices(stop)) else: # Literal values keys = key for key in keys: # TODO! - Some copy paste here from above. try: data[key] except (KeyError, IndexError): if context['on_missing'] == c.ON_MISSING_CONTINUE: continue elif context['on_missing'] == c.ON_MISSING_CREATE: data = _auto_fill(data, data_type, key, context['path_parts'], path_pos) else: raise instruction = _walk_path( context, data=data[key], key=key, parent=data, path_pos=path_pos + 1, path=path.add((c.KEY_LITERAL | data_type, key, path_pos))) if instruction == c.WALK_PRUNE: break if instruction == c.WALK_TERMINATE: return c.WALK_TERMINATE else: raise Exception('Bad key type')
def _walk_path(context, data, path_pos, parent, key, path): data_type = guess_type(data) instruction = context['function']( # Things which change all the time data=data, data_type=data_type, path_pos=path_pos, parent=parent, key=key, path=path, # Things we calculate to be nice terminal=path_pos == len(context['path_parts']), **context) if instruction != c.WALK_CONTINUE: return instruction # We've run out of path if path_pos >= len(context['path_parts']): return key_type, key = context['path_parts'][path_pos] # Type mismatch! if not key_type & data_type: if context['on_mismatch'] == c.ON_MISMATCH_FAIL: raise ValueError( 'Expected %s but found %s at %s: %s' % (c.STRINGS[key_type & c.TYPE_MASK], c.STRINGS[data_type], data, canonical_path(context['path_parts']))) elif context['on_mismatch'] == c.ON_MISMATCH_CONTINUE: return c.WALK_CONTINUE else: raise Exception('wut?') # It's a super mad recursion into the data structure elif key_type & c.KEY_RECURSE: recurse_context = dict(context, on_mismatch=c.ON_MISMATCH_CONTINUE) if key_type & c.KEY_WILD: return walk( data=data, function=lambda **kwargs: _walk_path(context=recurse_context, data=kwargs['data'], path_pos=path_pos + 1, parent=kwargs['parent'], key=None, path=kwargs['path']), parent=parent, key=None, path=path) elif key_type & c.KEY_LITERAL: return walk( data=data, function=lambda **kwargs: _walk_path(context=recurse_context, data=kwargs['data'], path_pos=path_pos + 1, parent=kwargs['parent'], key=None, path=kwargs['path']) if kwargs['key'] == key else c.WALK_CONTINUE, parent=parent, key=None, path=path) else: raise ValueError('HOW DO YOU EVEN') # A literal, we know exactly where to go elif key_type & c.KEY_LITERAL: missing = (key_type & c.TYPE_DICT and key not in data) or (key_type & c.TYPE_LIST and key >= len(data)) if missing: if context['on_missing'] == c.ON_MISSING_CONTINUE: return elif context['on_missing'] == c.ON_MISSING_CREATE: data = _auto_fill(data, data_type, key, context['path_parts'], path_pos) else: raise Exception('wut?') return _walk_path(context, data=data[key], key=key, parent=data, path=path.add((c.KEY_LITERAL | data_type, key)), path_pos=path_pos + 1) # A wild card, we go everywhere elif key_type & c.KEY_WILD: if data_type & c.TYPE_LIST: keys = xrange(len(data)) elif data_type & c.TYPE_DICT: keys = data.iterkeys() else: raise ValueError('Unknown sub-object type') for key in keys: instruction = _walk_path(context, data=data[key], key=key, parent=data, path_pos=path_pos + 1, path=path.add( (c.KEY_LITERAL | data_type, key))) if instruction == c.WALK_PRUNE: break if instruction == c.WALK_TERMINATE: return c.WALK_TERMINATE else: raise Exception('Bad key type')