def _initialize(self, data, txid):
        # separate external children data away from locally stored data
        # based on child_node annotations in protobuf
        children = {}
        for field_name, field in children_fields(self._type).iteritems():
            field_value = getattr(data, field_name)
            if field.is_container:
                if field.key:
                    keys_seen = set()
                    children[field_name] = lst = []
                    for v in field_value:
                        rev = self._mknode(v, txid=txid).latest
                        key = getattr(v, field.key)
                        if key in keys_seen:
                            raise ValueError('Duplicate key "{}"'.format(key))
                        lst.append(rev)
                        keys_seen.add(key)
                else:
                    children[field_name] = [
                        self._mknode(v, txid=txid).latest for v in field_value]
            else:
                children[field_name] = [
                    self._mknode(field_value, txid=txid).latest]
            data.ClearField(field_name)

        branch = ConfigBranch(self, auto_prune=self._auto_prune)
        rev = self._mkrev(branch, data, children)
        self._make_latest(branch, rev)
        self._branches[txid] = branch
Exemple #2
0
    def _initialize(self, data, txid):
        # separate external children data away from locally stored data
        # based on child_node annotations in protobuf
        children = {}
        for field_name, field in children_fields(self._type).iteritems():
            field_value = getattr(data, field_name)
            if field.is_container:
                if field.key:
                    keys_seen = set()
                    children[field_name] = lst = []
                    for v in field_value:
                        rev = self._mknode(v, txid=txid).latest
                        key = getattr(v, field.key)
                        if key in keys_seen:
                            raise ValueError('Duplicate key "{}"'.format(key))
                        lst.append(rev)
                        keys_seen.add(key)
                else:
                    children[field_name] = [
                        self._mknode(v, txid=txid).latest for v in field_value
                    ]
            else:
                children[field_name] = [
                    self._mknode(field_value, txid=txid).latest
                ]
            data.ClearField(field_name)

        branch = ConfigBranch(self, auto_prune=self._auto_prune)
        rev = self._mkrev(branch, data, children)
        self._make_latest(branch, rev)
        self._branches[txid] = branch
    def _get_proxy(self, path, root, full_path, exclusive):
        while path.startswith('/'):
            path = path[1:]
        if not path:
            return self._mk_proxy(root, full_path, exclusive)

        # need to escalate
        rev = self._branches[None]._latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                raise ValueError('Cannot proxy a container field')
            if field.key:
                key, _, path = path.partition('/')
                key = field.key_from_str(key)
                children = rev._children[name]
                _, child_rev = find_rev_by_key(children, field.key, key)
                child_node = child_rev.node
                return child_node._get_proxy(path, root, full_path, exclusive)

            raise ValueError('Cannot index into container with no keys')

        else:
            child_rev = rev._children[name][0]
            child_node = child_rev.node
            return child_node._get_proxy(path, root, full_path, exclusive)
Exemple #4
0
    def _get_proxy(self, path, root, full_path, exclusive):
        while path.startswith('/'):
            path = path[1:]
        if not path:
            return self._mk_proxy(root, full_path, exclusive)

        # need to escalate
        rev = self._branches[None]._latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                raise ValueError('Cannot proxy a container field')
            if field.key:
                key, _, path = path.partition('/')
                key = field.key_from_str(key)
                children = rev._children[name]
                _, child_rev = find_rev_by_key(children, field.key, key)
                child_node = child_rev.node
                return child_node._get_proxy(path, root, full_path, exclusive)

            raise ValueError('Cannot index into container with no keys')

        else:
            child_rev = rev._children[name][0]
            child_node = child_rev.node
            return child_node._get_proxy(path, root, full_path, exclusive)
Exemple #5
0
    def update(self, path, data, strict=False, txid=None, mk_branch=None):

        while path.startswith('/'):
            path = path[1:]

        try:
            branch = self._branches[txid]
        except KeyError:
            branch = mk_branch(self)

        if not path:
            return self._do_update(branch, data, strict)

        rev = branch._latest  # change is always made to the latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                raise ValueError('Cannot update a list')
            if field.key:
                key, _, path = path.partition('/')
                key = field.key_from_str(key)
                children = copy(rev._children[name])
                idx, child_rev = find_rev_by_key(children, field.key, key)
                child_node = child_rev.node
                # chek if deep copy will work better
                new_child_rev = child_node.update(path, data, strict, txid,
                                                  mk_branch)
                if new_child_rev.hash == child_rev.hash:
                    # When the new_child_rev goes out of scope,
                    # it's destructor gets invoked as it is not being
                    # referred by any other data structures.  To prevent
                    # this to trigger the hash it is holding from being
                    # erased in the db, its hash is set to None.  If the
                    # new_child_rev object is pointing at the same address
                    # as the child_rev address then do not clear the hash
                    if new_child_rev != child_rev:
                        log.debug('clear-hash',
                                  hash=new_child_rev.hash,
                                  object_ref=new_child_rev)
                        new_child_rev.clear_hash()
                    return branch._latest
                if getattr(new_child_rev.data, field.key) != key:
                    raise ValueError('Cannot change key field')
                children[idx] = new_child_rev
                rev = rev.update_children(name, children, branch)
                self._make_latest(branch, rev)
                return rev
            else:
                raise ValueError('Cannot index into container with no keys')

        else:
            child_rev = rev._children[name][0]
            child_node = child_rev.node
            new_child_rev = child_node.update(path, data, strict, txid,
                                              mk_branch)
            rev = rev.update_children(name, [new_child_rev], branch)
            self._make_latest(branch, rev)
            return rev
    def update(self, path, data, strict=False, txid=None, mk_branch=None):

        while path.startswith('/'):
            path = path[1:]

        try:
            branch = self._branches[txid]
        except KeyError:
            branch = mk_branch(self)

        if not path:
            return self._do_update(branch, data, strict)

        rev = branch._latest  # change is always made to the latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                raise ValueError('Cannot update a list')
            if field.key:
                key, _, path = path.partition('/')
                key = field.key_from_str(key)
                children = copy(rev._children[name])
                idx, child_rev = find_rev_by_key(children, field.key, key)
                child_node = child_rev.node
                # chek if deep copy will work better
                new_child_rev = child_node.update(
                    path, data, strict, txid, mk_branch)
                if new_child_rev.hash == child_rev.hash:
                    # When the new_child_rev goes out of scope,
                    # it's destructor gets invoked as it is not being
                    # referred by any other data structures.  To prevent
                    # this to trigger the hash it is holding from being
                    # erased in the db, its hash is set to None.  If the
                    # new_child_rev object is pointing at the same address
                    # as the child_rev address then do not clear the hash
                    if new_child_rev != child_rev:
                        log.debug('clear-hash',
                             hash=new_child_rev.hash, object_ref=new_child_rev)
                        new_child_rev.clear_hash()
                    return branch._latest
                if getattr(new_child_rev.data, field.key) != key:
                    raise ValueError('Cannot change key field')
                children[idx] = new_child_rev
                rev = rev.update_children(name, children, branch)
                self._make_latest(branch, rev)
                return rev
            else:
                raise ValueError('Cannot index into container with no keys')

        else:
            child_rev = rev._children[name][0]
            child_node = child_rev.node
            new_child_rev = child_node.update(
                path, data, strict, txid, mk_branch)
            rev = rev.update_children(name, [new_child_rev], branch)
            self._make_latest(branch, rev)
            return rev
Exemple #7
0
    def add(self, path, data, txid=None, mk_branch=None):
        while path.startswith('/'):
            path = path[1:]
        if not path:
            raise ValueError('Cannot add to non-container node')

        try:
            branch = self._branches[txid]
        except KeyError:
            branch = mk_branch(self)

        rev = branch._latest  # change is always made to latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                # we do need to add a new child to the field
                if field.key:
                    if self._proxy is not None:
                        self._proxy.invoke_callbacks(CallbackType.PRE_ADD,
                                                     data)
                    children = copy(rev._children[name])
                    key = getattr(data, field.key)
                    try:
                        find_rev_by_key(children, field.key, key)
                    except KeyError:
                        pass
                    else:
                        raise ValueError('Duplicate key "{}"'.format(key))
                    child_rev = self._mknode(data).latest
                    children.append(child_rev)
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev,
                                      ((CallbackType.POST_ADD, data), ))
                    return rev
                else:
                    # adding to non-keyed containers not implemented yet
                    raise ValueError('Cannot add to non-keyed container')
            else:
                if field.key:
                    # need to escalate
                    key, _, path = path.partition('/')
                    key = field.key_from_str(key)
                    children = copy(rev._children[name])
                    idx, child_rev = find_rev_by_key(children, field.key, key)
                    child_node = child_rev.node
                    new_child_rev = child_node.add(path, data, txid, mk_branch)
                    children[idx] = new_child_rev
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev)
                    return rev
                else:
                    raise ValueError(
                        'Cannot index into container with no keys')
        else:
            raise ValueError('Cannot add to non-container field')
    def add(self, path, data, txid=None, mk_branch=None):
        while path.startswith('/'):
            path = path[1:]
        if not path:
            raise ValueError('Cannot add to non-container node')

        try:
            branch = self._branches[txid]
        except KeyError:
            branch = mk_branch(self)

        rev = branch._latest  # change is always made to latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                # we do need to add a new child to the field
                if field.key:
                    if self._proxy is not None:
                        self._proxy.invoke_callbacks(
                            CallbackType.PRE_ADD, data)
                    children = copy(rev._children[name])
                    key = getattr(data, field.key)
                    try:
                        find_rev_by_key(children, field.key, key)
                    except KeyError:
                        pass
                    else:
                        raise ValueError('Duplicate key "{}"'.format(key))
                    child_rev = self._mknode(data).latest
                    children.append(child_rev)
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev,
                                      ((CallbackType.POST_ADD, data),))
                    return rev
                else:
                    # adding to non-keyed containers not implemented yet
                    raise ValueError('Cannot add to non-keyed container')
            else:
                if field.key:
                    # need to escalate
                    key, _, path = path.partition('/')
                    key = field.key_from_str(key)
                    children = copy(rev._children[name])
                    idx, child_rev = find_rev_by_key(children, field.key, key)
                    child_node = child_rev.node
                    new_child_rev = child_node.add(path, data, txid, mk_branch)
                    children[idx] = new_child_rev
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev)
                    return rev
                else:
                    raise ValueError(
                        'Cannot index into container with no keys')
        else:
            raise ValueError('Cannot add to non-container field')
Exemple #9
0
 def _test_no_children(self, data):
     for field_name, field in children_fields(self._type).items():
         field_value = getattr(data, field_name)
         if field.is_container:
             if len(field_value):
                 raise NotImplementedError(
                     'Cannot update external children')
         else:
             if data.HasField(field_name):
                 raise NotImplementedError(
                     'Cannot update externel children')
 def _test_no_children(self, data):
     for field_name, field in children_fields(self._type).items():
         field_value = getattr(data, field_name)
         if field.is_container:
             if len(field_value):
                 raise NotImplementedError(
                     'Cannot update external children')
         else:
             if data.HasField(field_name):
                 raise NotImplementedError(
                     'Cannot update externel children')
Exemple #11
0
    def remove(self, path, txid=None, mk_branch=None):
        while path.startswith('/'):
            path = path[1:]
        if not path:
            raise ValueError('Cannot remove from non-container node')

        try:
            branch = self._branches[txid]
        except KeyError:
            branch = mk_branch(self)

        rev = branch._latest  # change is always made to latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                raise ValueError("Cannot remove without a key")
            if field.key:
                key, _, path = path.partition('/')
                key = field.key_from_str(key)
                if path:
                    # need to escalate
                    children = copy(rev._children[name])
                    idx, child_rev = find_rev_by_key(children, field.key, key)
                    child_node = child_rev.node
                    new_child_rev = child_node.remove(path, txid, mk_branch)
                    children[idx] = new_child_rev
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev)
                    return rev
                else:
                    # need to remove from this very node
                    children = copy(rev._children[name])
                    idx, child_rev = find_rev_by_key(children, field.key, key)
                    if self._proxy is not None:
                        data = child_rev.data
                        self._proxy.invoke_callbacks(CallbackType.PRE_REMOVE,
                                                     data)
                        post_anno = ((CallbackType.POST_REMOVE, data), )
                    else:
                        post_anno = ((CallbackType.POST_REMOVE,
                                      child_rev.data), )
                    del children[idx]
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev, post_anno)
                    return rev
            else:
                raise ValueError('Cannot remove from non-keyed container')
        else:
            raise ValueError('Cannot remove non-conatiner field')
    def remove(self, path, txid=None, mk_branch=None):
        while path.startswith('/'):
            path = path[1:]
        if not path:
            raise ValueError('Cannot remove from non-container node')

        try:
            branch = self._branches[txid]
        except KeyError:
            branch = mk_branch(self)

        rev = branch._latest  # change is always made to latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                raise ValueError("Cannot remove without a key")
            if field.key:
                key, _, path = path.partition('/')
                key = field.key_from_str(key)
                if path:
                    # need to escalate
                    children = copy(rev._children[name])
                    idx, child_rev = find_rev_by_key(children, field.key, key)
                    child_node = child_rev.node
                    new_child_rev = child_node.remove(path, txid, mk_branch)
                    children[idx] = new_child_rev
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev)
                    return rev
                else:
                    # need to remove from this very node
                    children = copy(rev._children[name])
                    idx, child_rev = find_rev_by_key(children, field.key, key)
                    if self._proxy is not None:
                        data = child_rev.data
                        self._proxy.invoke_callbacks(
                            CallbackType.PRE_REMOVE, data)
                        post_anno = ((CallbackType.POST_REMOVE, data),)
                    else:
                        post_anno = ((CallbackType.POST_REMOVE, child_rev.data),)
                    del children[idx]
                    rev = rev.update_children(name, children, branch)
                    self._make_latest(branch, rev, post_anno)
                    return rev
            else:
                raise ValueError('Cannot remove from non-keyed container')
        else:
            raise ValueError('Cannot remove non-conatiner field')
Exemple #13
0
    def update(self, path, data, strict=False, txid=None, mk_branch=None):

        while path.startswith('/'):
            path = path[1:]

        try:
            branch = self._branches[txid]
        except KeyError:
            branch = mk_branch(self)

        if not path:
            return self._do_update(branch, data, strict)

        rev = branch._latest  # change is always made to the latest
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if not path:
                raise ValueError('Cannot update a list')
            if field.key:
                key, _, path = path.partition('/')
                key = field.key_from_str(key)
                children = copy(rev._children[name])
                idx, child_rev = find_rev_by_key(children, field.key, key)
                child_node = child_rev.node
                new_child_rev = child_node.update(path, data, strict, txid,
                                                  mk_branch)
                if new_child_rev.hash == child_rev.hash:
                    # no change, we can return
                    return branch._latest
                if getattr(new_child_rev.data, field.key) != key:
                    raise ValueError('Cannot change key field')
                children[idx] = new_child_rev
                rev = rev.update_children(name, children, branch)
                self._make_latest(branch, rev)
                return rev
            else:
                raise ValueError('Cannot index into container with no keys')

        else:
            child_rev = rev._children[name][0]
            child_node = child_rev.node
            new_child_rev = child_node.update(path, data, strict, txid,
                                              mk_branch)
            rev = rev.update_children(name, [new_child_rev], branch)
            self._make_latest(branch, rev)
            return rev
Exemple #14
0
    def _get(self, rev, path, depth):

        if not path:
            return self._do_get(rev, depth)

        # ... otherwise
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if field.key:
                children = rev._children[name]
                if path:
                    # need to escalate further
                    key, _, path = path.partition('/')
                    key = field.key_from_str(key)
                    _, child_rev = find_rev_by_key(children, field.key, key)
                    child_node = child_rev.node
                    return child_node._get(child_rev, path, depth)
                else:
                    # we are the node of interest
                    response = []
                    for child_rev in children:
                        child_node = child_rev.node
                        value = child_node._do_get(child_rev, depth)
                        response.append(value)
                    return response
            else:
                if path:
                    raise LookupError(
                        'Cannot index into container with no key defined')
                response = []
                for child_rev in rev._children[name]:
                    child_node = child_rev.node
                    value = child_node._do_get(child_rev, depth)
                    response.append(value)
                return response
        else:
            child_rev = rev._children[name][0]
            child_node = child_rev.node
            return child_node._get(child_rev, path, depth)
    def _get(self, rev, path, depth):

        if not path:
            return self._do_get(rev, depth)

        # ... otherwise
        name, _, path = path.partition('/')
        field = children_fields(self._type)[name]
        if field.is_container:
            if field.key:
                children = rev._children[name]
                if path:
                    # need to escalate further
                    key, _, path = path.partition('/')
                    key = field.key_from_str(key)
                    _, child_rev = find_rev_by_key(children, field.key, key)
                    child_node = child_rev.node
                    return child_node._get(child_rev, path, depth)
                else:
                    # we are the node of interest
                    response = []
                    for child_rev in children:
                        child_node = child_rev.node
                        value = child_node._do_get(child_rev, depth)
                        response.append(value)
                    return response
            else:
                if path:
                    raise LookupError(
                        'Cannot index into container with no key defined')
                response = []
                for child_rev in rev._children[name]:
                    child_node = child_rev.node
                    value = child_node._do_get(child_rev, depth)
                    response.append(value)
                return response
        else:
            child_rev = rev._children[name][0]
            child_node = child_rev.node
            return child_node._get(child_rev, path, depth)
Exemple #16
0
    def load(cls, branch, kv_store, msg_cls, hash):
        blob = kv_store[hash]
        if cls.compress:
            blob = decompress(blob)
        data = loads(blob)

        config_hash = data['config']
        config_data = cls.load_config(kv_store, msg_cls, config_hash)

        children_list = data['children']
        assembled_children = {}
        node = branch._node
        for field_name, meta in children_fields(msg_cls).iteritems():
            child_msg_cls = tmp_cls_loader(meta.module, meta.type)
            children = []
            for child_hash in children_list[field_name]:
                child_node = node._mknode(child_msg_cls)
                child_node.load_latest(child_hash)
                child_rev = child_node.latest
                children.append(child_rev)
            assembled_children[field_name] = children
        rev = cls(branch, config_data, assembled_children)
        return rev
    def load(cls, branch, kv_store, msg_cls, hash):
        #  Update the branch's config store
        blob = kv_store[hash]
        if cls.compress:
            blob = decompress(blob)
        data = loads(blob)

        config_hash = data['config']
        config_data = cls.load_config(kv_store, msg_cls, config_hash)

        children_list = data['children']
        assembled_children = {}
        node = branch._node
        for field_name, meta in children_fields(msg_cls).iteritems():
            child_msg_cls = tmp_cls_loader(meta.module, meta.type)
            children = []
            for child_hash in children_list[field_name]:
                child_node = node._mknode(child_msg_cls)
                child_node.load_latest(child_hash)
                child_rev = child_node.latest
                children.append(child_rev)
            assembled_children[field_name] = children
        rev = cls(branch, config_data, assembled_children)
        return rev
Exemple #18
0
def merge_3way(fork_rev, src_rev, dst_rev, merge_child_func, dry_run=False):
    """
    Attempt to merge src_rev into dst_rev but taking into account what have
    changed in both revs since the last known common point, the fork_rev.
    In case of conflict, raise a MergeConflictException(). If dry run is True,
    don't actually perform the merge, but detect potential conflicts.

    This function recurses into all children nodes stored under the rev and
    performs the merge if the children is also part of a transaction branch.

    :param fork_rev: Point of forking (last known common state between branches
    :param src_rev: Latest rev from which we merge to dst_rev
    :param dst_rev: Target (destination) rev
    :param merge_child_fun: To run a potential merge in all children that
    may need merge (determined from the local changes)
    :param dry_run: If True, do not perform the merge, but detect merge
    conflicts.
    :return: The new dst_rev (a new rev instance) the list of changes that
    occurred in this node or any of its children as part of this merge.
    """

    # to collect change tuples of (<callback-type>, <op-context>)
    changes = []

    class AnalyzeChanges(object):
        def __init__(self, lst1, lst2, keyname):
            self.keymap1 = OrderedDict((getattr(rev._config._data, keyname), i)
                                       for i, rev in enumerate(lst1))
            self.keymap2 = OrderedDict((getattr(rev._config._data, keyname), i)
                                       for i, rev in enumerate(lst2))
            self.added_keys = [
                k for k in self.keymap2.iterkeys() if k not in self.keymap1]
            self.removed_keys = [
                k for k in self.keymap1.iterkeys() if k not in self.keymap2]
            self.changed_keys = [
                k for k in self.keymap1.iterkeys()
                if k in self.keymap2 and
                    lst1[self.keymap1[k]]._hash != lst2[self.keymap2[k]]._hash
            ]

    # Note: there are a couple of special cases that can be optimized
    # for larer on. But since premature optimization is a bad idea, we
    # defer them.

    # deal with config data first
    if dst_rev._config is fork_rev._config:
        # no change in master, accept src if different
        config_changed = dst_rev._config != src_rev._config
    else:
        if dst_rev._config.hash != src_rev._config.hash:
            raise MergeConflictException('Config collision')
        config_changed = True

    # now to the external children fields
    new_children = dst_rev._children.copy()
    _children_fields = children_fields(fork_rev.data.__class__)

    for field_name, field in _children_fields.iteritems():

        fork_list = fork_rev._children[field_name]
        src_list = src_rev._children[field_name]
        dst_list = dst_rev._children[field_name]

        if dst_list == src_list:
            # we do not need to change the dst, however we still need
            # to complete the branch purging in child nodes so not
            # to leave dangling branches around
            [merge_child_func(rev) for rev in src_list]
            continue

        if not field.key:
            # If the list is not keyed, we really should not merge. We merely
            # check for collision, i.e., if both changed (and not same)
            if dst_list == fork_list:
                # dst branch did not change since fork

                assert src_list != fork_list, 'We should not be here otherwise'

                # the incoming (src) rev changed, and we have to apply it
                new_children[field_name] = [
                    merge_child_func(rev) for rev in src_list]

                if field.is_container:
                    changes.append((CallbackType.POST_LISTCHANGE,
                                    OperationContext(field_name=field_name)))

            else:
                if src_list != fork_list:
                    raise MergeConflictException(
                        'Cannot merge because single child node or un-keyed'
                        'children list has changed')

        else:

            if dst_list == fork_list:
                # Destination did not change

                # We need to analyze only the changes on the incoming rev
                # since fork
                src = AnalyzeChanges(fork_list, src_list, field.key)

                new_list = copy(src_list)  # we start from the source list

                for key in src.added_keys:
                    idx = src.keymap2[key]
                    new_rev = merge_child_func(new_list[idx])
                    new_list[idx] = new_rev
                    changes.append(
                        (CallbackType.POST_ADD,
                         new_rev.data))
                         # OperationContext(
                         #     field_name=field_name,
                         #     child_key=key,
                         #     data=new_rev.data)))

                for key in src.removed_keys:
                    old_rev = fork_list[src.keymap1[key]]
                    changes.append((
                        CallbackType.POST_REMOVE,
                        old_rev.data))
                        # OperationContext(
                        #     field_name=field_name,
                        #     child_key=key,
                        #     data=old_rev.data)))

                for key in src.changed_keys:
                    idx = src.keymap2[key]
                    new_rev = merge_child_func(new_list[idx])
                    new_list[idx] = new_rev
                    # updated child gets its own change event

                new_children[field_name] = new_list

            else:

                # For keyed fields we can really investigate what has been
                # added, removed, or changed in both branches and do a
                # fine-grained collision detection and merge

                src = AnalyzeChanges(fork_list, src_list, field.key)
                dst = AnalyzeChanges(fork_list, dst_list, field.key)

                new_list = copy(dst_list)  # this time we start with the dst

                for key in src.added_keys:
                    # we cannot add if it has been added and is different
                    if key in dst.added_keys:
                        # it has been added to both, we need to check if
                        # they are the same
                        child_dst_rev = dst_list[dst.keymap2[key]]
                        child_src_rev = src_list[src.keymap2[key]]
                        if child_dst_rev.hash == child_src_rev.hash:
                            # they match, so we do not need to change the
                            # dst list, but we still need to purge the src
                            # branch
                            merge_child_func(child_dst_rev)
                        else:
                            raise MergeConflictException(
                                'Cannot add because it has been added and '
                                'different'
                            )
                    else:
                        # this is a brand new key, need to add it
                        new_rev = merge_child_func(src_list[src.keymap2[key]])
                        new_list.append(new_rev)
                        changes.append((
                            CallbackType.POST_ADD,
                            new_rev.data))
                            # OperationContext(
                            #     field_name=field_name,
                            #     child_key=key,
                            #     data=new_rev.data)))

                for key in src.changed_keys:
                    # we cannot change if it was removed in dst
                    if key in dst.removed_keys:
                        raise MergeConflictException(
                            'Cannot change because it has been removed')

                    # if it changed in dst as well, we need to check if they
                    # match (same change
                    elif key in dst.changed_keys:
                        child_dst_rev = dst_list[dst.keymap2[key]]
                        child_src_rev = src_list[src.keymap2[key]]
                        if child_dst_rev.hash == child_src_rev.hash:
                            # they match, so we do not need to change the
                            # dst list, but we still need to purge the src
                            # branch
                            merge_child_func(child_src_rev)
                        elif child_dst_rev._config.hash != child_src_rev._config.hash:
                            raise MergeConflictException(
                                'Cannot update because it has been changed and '
                                'different'
                            )
                        else:
                            new_rev = merge_child_func(
                                src_list[src.keymap2[key]])
                            new_list[dst.keymap2[key]] = new_rev
                            # no announcement for child update

                    else:
                        # it only changed in src branch
                        new_rev = merge_child_func(src_list[src.keymap2[key]])
                        new_list[dst.keymap2[key]] = new_rev
                        # no announcement for child update

                for key in reversed(src.removed_keys):  # we go from highest
                                                        # index to lowest

                    # we cannot remove if it has changed in dst
                    if key in dst.changed_keys:
                        raise MergeConflictException(
                            'Cannot remove because it has changed')

                    # if it has not been removed yet from dst, then remove it
                    if key not in dst.removed_keys:
                        dst_idx = dst.keymap2[key]
                        old_rev = new_list.pop(dst_idx)
                        changes.append((
                            CallbackType.POST_REMOVE,
                            old_rev.data))
                            # OperationContext(
                            #     field_name=field_name,
                            #     child_key=key,
                            #     data=old_rev.data)))

                new_children[field_name] = new_list

    if not dry_run:
        rev = src_rev if config_changed else dst_rev
        rev = rev.update_all_children(new_children, dst_rev._branch)
        if config_changed:
            changes.append((CallbackType.POST_UPDATE, rev.data))
        return rev, changes

    else:
        return None, None