def _initialize(self, data, txid): # separate external children data away from locally stored data # based on child_node annotations in protobuf children = {} for field_name, field in children_fields(self._type).iteritems(): field_value = getattr(data, field_name) if field.is_container: if field.key: keys_seen = set() children[field_name] = lst = [] for v in field_value: rev = self._mknode(v, txid=txid).latest key = getattr(v, field.key) if key in keys_seen: raise ValueError('Duplicate key "{}"'.format(key)) lst.append(rev) keys_seen.add(key) else: children[field_name] = [ self._mknode(v, txid=txid).latest for v in field_value] else: children[field_name] = [ self._mknode(field_value, txid=txid).latest] data.ClearField(field_name) branch = ConfigBranch(self, auto_prune=self._auto_prune) rev = self._mkrev(branch, data, children) self._make_latest(branch, rev) self._branches[txid] = branch
def _initialize(self, data, txid): # separate external children data away from locally stored data # based on child_node annotations in protobuf children = {} for field_name, field in children_fields(self._type).iteritems(): field_value = getattr(data, field_name) if field.is_container: if field.key: keys_seen = set() children[field_name] = lst = [] for v in field_value: rev = self._mknode(v, txid=txid).latest key = getattr(v, field.key) if key in keys_seen: raise ValueError('Duplicate key "{}"'.format(key)) lst.append(rev) keys_seen.add(key) else: children[field_name] = [ self._mknode(v, txid=txid).latest for v in field_value ] else: children[field_name] = [ self._mknode(field_value, txid=txid).latest ] data.ClearField(field_name) branch = ConfigBranch(self, auto_prune=self._auto_prune) rev = self._mkrev(branch, data, children) self._make_latest(branch, rev) self._branches[txid] = branch
def _get_proxy(self, path, root, full_path, exclusive): while path.startswith('/'): path = path[1:] if not path: return self._mk_proxy(root, full_path, exclusive) # need to escalate rev = self._branches[None]._latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: raise ValueError('Cannot proxy a container field') if field.key: key, _, path = path.partition('/') key = field.key_from_str(key) children = rev._children[name] _, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node return child_node._get_proxy(path, root, full_path, exclusive) raise ValueError('Cannot index into container with no keys') else: child_rev = rev._children[name][0] child_node = child_rev.node return child_node._get_proxy(path, root, full_path, exclusive)
def _get_proxy(self, path, root, full_path, exclusive): while path.startswith('/'): path = path[1:] if not path: return self._mk_proxy(root, full_path, exclusive) # need to escalate rev = self._branches[None]._latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: raise ValueError('Cannot proxy a container field') if field.key: key, _, path = path.partition('/') key = field.key_from_str(key) children = rev._children[name] _, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node return child_node._get_proxy(path, root, full_path, exclusive) raise ValueError('Cannot index into container with no keys') else: child_rev = rev._children[name][0] child_node = child_rev.node return child_node._get_proxy(path, root, full_path, exclusive)
def update(self, path, data, strict=False, txid=None, mk_branch=None): while path.startswith('/'): path = path[1:] try: branch = self._branches[txid] except KeyError: branch = mk_branch(self) if not path: return self._do_update(branch, data, strict) rev = branch._latest # change is always made to the latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: raise ValueError('Cannot update a list') if field.key: key, _, path = path.partition('/') key = field.key_from_str(key) children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node # chek if deep copy will work better new_child_rev = child_node.update(path, data, strict, txid, mk_branch) if new_child_rev.hash == child_rev.hash: # When the new_child_rev goes out of scope, # it's destructor gets invoked as it is not being # referred by any other data structures. To prevent # this to trigger the hash it is holding from being # erased in the db, its hash is set to None. If the # new_child_rev object is pointing at the same address # as the child_rev address then do not clear the hash if new_child_rev != child_rev: log.debug('clear-hash', hash=new_child_rev.hash, object_ref=new_child_rev) new_child_rev.clear_hash() return branch._latest if getattr(new_child_rev.data, field.key) != key: raise ValueError('Cannot change key field') children[idx] = new_child_rev rev = rev.update_children(name, children, branch) self._make_latest(branch, rev) return rev else: raise ValueError('Cannot index into container with no keys') else: child_rev = rev._children[name][0] child_node = child_rev.node new_child_rev = child_node.update(path, data, strict, txid, mk_branch) rev = rev.update_children(name, [new_child_rev], branch) self._make_latest(branch, rev) return rev
def update(self, path, data, strict=False, txid=None, mk_branch=None): while path.startswith('/'): path = path[1:] try: branch = self._branches[txid] except KeyError: branch = mk_branch(self) if not path: return self._do_update(branch, data, strict) rev = branch._latest # change is always made to the latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: raise ValueError('Cannot update a list') if field.key: key, _, path = path.partition('/') key = field.key_from_str(key) children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node # chek if deep copy will work better new_child_rev = child_node.update( path, data, strict, txid, mk_branch) if new_child_rev.hash == child_rev.hash: # When the new_child_rev goes out of scope, # it's destructor gets invoked as it is not being # referred by any other data structures. To prevent # this to trigger the hash it is holding from being # erased in the db, its hash is set to None. If the # new_child_rev object is pointing at the same address # as the child_rev address then do not clear the hash if new_child_rev != child_rev: log.debug('clear-hash', hash=new_child_rev.hash, object_ref=new_child_rev) new_child_rev.clear_hash() return branch._latest if getattr(new_child_rev.data, field.key) != key: raise ValueError('Cannot change key field') children[idx] = new_child_rev rev = rev.update_children(name, children, branch) self._make_latest(branch, rev) return rev else: raise ValueError('Cannot index into container with no keys') else: child_rev = rev._children[name][0] child_node = child_rev.node new_child_rev = child_node.update( path, data, strict, txid, mk_branch) rev = rev.update_children(name, [new_child_rev], branch) self._make_latest(branch, rev) return rev
def add(self, path, data, txid=None, mk_branch=None): while path.startswith('/'): path = path[1:] if not path: raise ValueError('Cannot add to non-container node') try: branch = self._branches[txid] except KeyError: branch = mk_branch(self) rev = branch._latest # change is always made to latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: # we do need to add a new child to the field if field.key: if self._proxy is not None: self._proxy.invoke_callbacks(CallbackType.PRE_ADD, data) children = copy(rev._children[name]) key = getattr(data, field.key) try: find_rev_by_key(children, field.key, key) except KeyError: pass else: raise ValueError('Duplicate key "{}"'.format(key)) child_rev = self._mknode(data).latest children.append(child_rev) rev = rev.update_children(name, children, branch) self._make_latest(branch, rev, ((CallbackType.POST_ADD, data), )) return rev else: # adding to non-keyed containers not implemented yet raise ValueError('Cannot add to non-keyed container') else: if field.key: # need to escalate key, _, path = path.partition('/') key = field.key_from_str(key) children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node new_child_rev = child_node.add(path, data, txid, mk_branch) children[idx] = new_child_rev rev = rev.update_children(name, children, branch) self._make_latest(branch, rev) return rev else: raise ValueError( 'Cannot index into container with no keys') else: raise ValueError('Cannot add to non-container field')
def add(self, path, data, txid=None, mk_branch=None): while path.startswith('/'): path = path[1:] if not path: raise ValueError('Cannot add to non-container node') try: branch = self._branches[txid] except KeyError: branch = mk_branch(self) rev = branch._latest # change is always made to latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: # we do need to add a new child to the field if field.key: if self._proxy is not None: self._proxy.invoke_callbacks( CallbackType.PRE_ADD, data) children = copy(rev._children[name]) key = getattr(data, field.key) try: find_rev_by_key(children, field.key, key) except KeyError: pass else: raise ValueError('Duplicate key "{}"'.format(key)) child_rev = self._mknode(data).latest children.append(child_rev) rev = rev.update_children(name, children, branch) self._make_latest(branch, rev, ((CallbackType.POST_ADD, data),)) return rev else: # adding to non-keyed containers not implemented yet raise ValueError('Cannot add to non-keyed container') else: if field.key: # need to escalate key, _, path = path.partition('/') key = field.key_from_str(key) children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node new_child_rev = child_node.add(path, data, txid, mk_branch) children[idx] = new_child_rev rev = rev.update_children(name, children, branch) self._make_latest(branch, rev) return rev else: raise ValueError( 'Cannot index into container with no keys') else: raise ValueError('Cannot add to non-container field')
def _test_no_children(self, data): for field_name, field in children_fields(self._type).items(): field_value = getattr(data, field_name) if field.is_container: if len(field_value): raise NotImplementedError( 'Cannot update external children') else: if data.HasField(field_name): raise NotImplementedError( 'Cannot update externel children')
def _test_no_children(self, data): for field_name, field in children_fields(self._type).items(): field_value = getattr(data, field_name) if field.is_container: if len(field_value): raise NotImplementedError( 'Cannot update external children') else: if data.HasField(field_name): raise NotImplementedError( 'Cannot update externel children')
def remove(self, path, txid=None, mk_branch=None): while path.startswith('/'): path = path[1:] if not path: raise ValueError('Cannot remove from non-container node') try: branch = self._branches[txid] except KeyError: branch = mk_branch(self) rev = branch._latest # change is always made to latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: raise ValueError("Cannot remove without a key") if field.key: key, _, path = path.partition('/') key = field.key_from_str(key) if path: # need to escalate children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node new_child_rev = child_node.remove(path, txid, mk_branch) children[idx] = new_child_rev rev = rev.update_children(name, children, branch) self._make_latest(branch, rev) return rev else: # need to remove from this very node children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) if self._proxy is not None: data = child_rev.data self._proxy.invoke_callbacks(CallbackType.PRE_REMOVE, data) post_anno = ((CallbackType.POST_REMOVE, data), ) else: post_anno = ((CallbackType.POST_REMOVE, child_rev.data), ) del children[idx] rev = rev.update_children(name, children, branch) self._make_latest(branch, rev, post_anno) return rev else: raise ValueError('Cannot remove from non-keyed container') else: raise ValueError('Cannot remove non-conatiner field')
def remove(self, path, txid=None, mk_branch=None): while path.startswith('/'): path = path[1:] if not path: raise ValueError('Cannot remove from non-container node') try: branch = self._branches[txid] except KeyError: branch = mk_branch(self) rev = branch._latest # change is always made to latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: raise ValueError("Cannot remove without a key") if field.key: key, _, path = path.partition('/') key = field.key_from_str(key) if path: # need to escalate children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node new_child_rev = child_node.remove(path, txid, mk_branch) children[idx] = new_child_rev rev = rev.update_children(name, children, branch) self._make_latest(branch, rev) return rev else: # need to remove from this very node children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) if self._proxy is not None: data = child_rev.data self._proxy.invoke_callbacks( CallbackType.PRE_REMOVE, data) post_anno = ((CallbackType.POST_REMOVE, data),) else: post_anno = ((CallbackType.POST_REMOVE, child_rev.data),) del children[idx] rev = rev.update_children(name, children, branch) self._make_latest(branch, rev, post_anno) return rev else: raise ValueError('Cannot remove from non-keyed container') else: raise ValueError('Cannot remove non-conatiner field')
def update(self, path, data, strict=False, txid=None, mk_branch=None): while path.startswith('/'): path = path[1:] try: branch = self._branches[txid] except KeyError: branch = mk_branch(self) if not path: return self._do_update(branch, data, strict) rev = branch._latest # change is always made to the latest name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if not path: raise ValueError('Cannot update a list') if field.key: key, _, path = path.partition('/') key = field.key_from_str(key) children = copy(rev._children[name]) idx, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node new_child_rev = child_node.update(path, data, strict, txid, mk_branch) if new_child_rev.hash == child_rev.hash: # no change, we can return return branch._latest if getattr(new_child_rev.data, field.key) != key: raise ValueError('Cannot change key field') children[idx] = new_child_rev rev = rev.update_children(name, children, branch) self._make_latest(branch, rev) return rev else: raise ValueError('Cannot index into container with no keys') else: child_rev = rev._children[name][0] child_node = child_rev.node new_child_rev = child_node.update(path, data, strict, txid, mk_branch) rev = rev.update_children(name, [new_child_rev], branch) self._make_latest(branch, rev) return rev
def _get(self, rev, path, depth): if not path: return self._do_get(rev, depth) # ... otherwise name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if field.key: children = rev._children[name] if path: # need to escalate further key, _, path = path.partition('/') key = field.key_from_str(key) _, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node return child_node._get(child_rev, path, depth) else: # we are the node of interest response = [] for child_rev in children: child_node = child_rev.node value = child_node._do_get(child_rev, depth) response.append(value) return response else: if path: raise LookupError( 'Cannot index into container with no key defined') response = [] for child_rev in rev._children[name]: child_node = child_rev.node value = child_node._do_get(child_rev, depth) response.append(value) return response else: child_rev = rev._children[name][0] child_node = child_rev.node return child_node._get(child_rev, path, depth)
def _get(self, rev, path, depth): if not path: return self._do_get(rev, depth) # ... otherwise name, _, path = path.partition('/') field = children_fields(self._type)[name] if field.is_container: if field.key: children = rev._children[name] if path: # need to escalate further key, _, path = path.partition('/') key = field.key_from_str(key) _, child_rev = find_rev_by_key(children, field.key, key) child_node = child_rev.node return child_node._get(child_rev, path, depth) else: # we are the node of interest response = [] for child_rev in children: child_node = child_rev.node value = child_node._do_get(child_rev, depth) response.append(value) return response else: if path: raise LookupError( 'Cannot index into container with no key defined') response = [] for child_rev in rev._children[name]: child_node = child_rev.node value = child_node._do_get(child_rev, depth) response.append(value) return response else: child_rev = rev._children[name][0] child_node = child_rev.node return child_node._get(child_rev, path, depth)
def load(cls, branch, kv_store, msg_cls, hash): blob = kv_store[hash] if cls.compress: blob = decompress(blob) data = loads(blob) config_hash = data['config'] config_data = cls.load_config(kv_store, msg_cls, config_hash) children_list = data['children'] assembled_children = {} node = branch._node for field_name, meta in children_fields(msg_cls).iteritems(): child_msg_cls = tmp_cls_loader(meta.module, meta.type) children = [] for child_hash in children_list[field_name]: child_node = node._mknode(child_msg_cls) child_node.load_latest(child_hash) child_rev = child_node.latest children.append(child_rev) assembled_children[field_name] = children rev = cls(branch, config_data, assembled_children) return rev
def load(cls, branch, kv_store, msg_cls, hash): # Update the branch's config store blob = kv_store[hash] if cls.compress: blob = decompress(blob) data = loads(blob) config_hash = data['config'] config_data = cls.load_config(kv_store, msg_cls, config_hash) children_list = data['children'] assembled_children = {} node = branch._node for field_name, meta in children_fields(msg_cls).iteritems(): child_msg_cls = tmp_cls_loader(meta.module, meta.type) children = [] for child_hash in children_list[field_name]: child_node = node._mknode(child_msg_cls) child_node.load_latest(child_hash) child_rev = child_node.latest children.append(child_rev) assembled_children[field_name] = children rev = cls(branch, config_data, assembled_children) return rev
def merge_3way(fork_rev, src_rev, dst_rev, merge_child_func, dry_run=False): """ Attempt to merge src_rev into dst_rev but taking into account what have changed in both revs since the last known common point, the fork_rev. In case of conflict, raise a MergeConflictException(). If dry run is True, don't actually perform the merge, but detect potential conflicts. This function recurses into all children nodes stored under the rev and performs the merge if the children is also part of a transaction branch. :param fork_rev: Point of forking (last known common state between branches :param src_rev: Latest rev from which we merge to dst_rev :param dst_rev: Target (destination) rev :param merge_child_fun: To run a potential merge in all children that may need merge (determined from the local changes) :param dry_run: If True, do not perform the merge, but detect merge conflicts. :return: The new dst_rev (a new rev instance) the list of changes that occurred in this node or any of its children as part of this merge. """ # to collect change tuples of (<callback-type>, <op-context>) changes = [] class AnalyzeChanges(object): def __init__(self, lst1, lst2, keyname): self.keymap1 = OrderedDict((getattr(rev._config._data, keyname), i) for i, rev in enumerate(lst1)) self.keymap2 = OrderedDict((getattr(rev._config._data, keyname), i) for i, rev in enumerate(lst2)) self.added_keys = [ k for k in self.keymap2.iterkeys() if k not in self.keymap1] self.removed_keys = [ k for k in self.keymap1.iterkeys() if k not in self.keymap2] self.changed_keys = [ k for k in self.keymap1.iterkeys() if k in self.keymap2 and lst1[self.keymap1[k]]._hash != lst2[self.keymap2[k]]._hash ] # Note: there are a couple of special cases that can be optimized # for larer on. But since premature optimization is a bad idea, we # defer them. # deal with config data first if dst_rev._config is fork_rev._config: # no change in master, accept src if different config_changed = dst_rev._config != src_rev._config else: if dst_rev._config.hash != src_rev._config.hash: raise MergeConflictException('Config collision') config_changed = True # now to the external children fields new_children = dst_rev._children.copy() _children_fields = children_fields(fork_rev.data.__class__) for field_name, field in _children_fields.iteritems(): fork_list = fork_rev._children[field_name] src_list = src_rev._children[field_name] dst_list = dst_rev._children[field_name] if dst_list == src_list: # we do not need to change the dst, however we still need # to complete the branch purging in child nodes so not # to leave dangling branches around [merge_child_func(rev) for rev in src_list] continue if not field.key: # If the list is not keyed, we really should not merge. We merely # check for collision, i.e., if both changed (and not same) if dst_list == fork_list: # dst branch did not change since fork assert src_list != fork_list, 'We should not be here otherwise' # the incoming (src) rev changed, and we have to apply it new_children[field_name] = [ merge_child_func(rev) for rev in src_list] if field.is_container: changes.append((CallbackType.POST_LISTCHANGE, OperationContext(field_name=field_name))) else: if src_list != fork_list: raise MergeConflictException( 'Cannot merge because single child node or un-keyed' 'children list has changed') else: if dst_list == fork_list: # Destination did not change # We need to analyze only the changes on the incoming rev # since fork src = AnalyzeChanges(fork_list, src_list, field.key) new_list = copy(src_list) # we start from the source list for key in src.added_keys: idx = src.keymap2[key] new_rev = merge_child_func(new_list[idx]) new_list[idx] = new_rev changes.append( (CallbackType.POST_ADD, new_rev.data)) # OperationContext( # field_name=field_name, # child_key=key, # data=new_rev.data))) for key in src.removed_keys: old_rev = fork_list[src.keymap1[key]] changes.append(( CallbackType.POST_REMOVE, old_rev.data)) # OperationContext( # field_name=field_name, # child_key=key, # data=old_rev.data))) for key in src.changed_keys: idx = src.keymap2[key] new_rev = merge_child_func(new_list[idx]) new_list[idx] = new_rev # updated child gets its own change event new_children[field_name] = new_list else: # For keyed fields we can really investigate what has been # added, removed, or changed in both branches and do a # fine-grained collision detection and merge src = AnalyzeChanges(fork_list, src_list, field.key) dst = AnalyzeChanges(fork_list, dst_list, field.key) new_list = copy(dst_list) # this time we start with the dst for key in src.added_keys: # we cannot add if it has been added and is different if key in dst.added_keys: # it has been added to both, we need to check if # they are the same child_dst_rev = dst_list[dst.keymap2[key]] child_src_rev = src_list[src.keymap2[key]] if child_dst_rev.hash == child_src_rev.hash: # they match, so we do not need to change the # dst list, but we still need to purge the src # branch merge_child_func(child_dst_rev) else: raise MergeConflictException( 'Cannot add because it has been added and ' 'different' ) else: # this is a brand new key, need to add it new_rev = merge_child_func(src_list[src.keymap2[key]]) new_list.append(new_rev) changes.append(( CallbackType.POST_ADD, new_rev.data)) # OperationContext( # field_name=field_name, # child_key=key, # data=new_rev.data))) for key in src.changed_keys: # we cannot change if it was removed in dst if key in dst.removed_keys: raise MergeConflictException( 'Cannot change because it has been removed') # if it changed in dst as well, we need to check if they # match (same change elif key in dst.changed_keys: child_dst_rev = dst_list[dst.keymap2[key]] child_src_rev = src_list[src.keymap2[key]] if child_dst_rev.hash == child_src_rev.hash: # they match, so we do not need to change the # dst list, but we still need to purge the src # branch merge_child_func(child_src_rev) elif child_dst_rev._config.hash != child_src_rev._config.hash: raise MergeConflictException( 'Cannot update because it has been changed and ' 'different' ) else: new_rev = merge_child_func( src_list[src.keymap2[key]]) new_list[dst.keymap2[key]] = new_rev # no announcement for child update else: # it only changed in src branch new_rev = merge_child_func(src_list[src.keymap2[key]]) new_list[dst.keymap2[key]] = new_rev # no announcement for child update for key in reversed(src.removed_keys): # we go from highest # index to lowest # we cannot remove if it has changed in dst if key in dst.changed_keys: raise MergeConflictException( 'Cannot remove because it has changed') # if it has not been removed yet from dst, then remove it if key not in dst.removed_keys: dst_idx = dst.keymap2[key] old_rev = new_list.pop(dst_idx) changes.append(( CallbackType.POST_REMOVE, old_rev.data)) # OperationContext( # field_name=field_name, # child_key=key, # data=old_rev.data))) new_children[field_name] = new_list if not dry_run: rev = src_rev if config_changed else dst_rev rev = rev.update_all_children(new_children, dst_rev._branch) if config_changed: changes.append((CallbackType.POST_UPDATE, rev.data)) return rev, changes else: return None, None