def _inc_path(self):
""":returns: The path of the next sibling of a given node path."""
newpos = self._str2int(self.path[-self.steplen:]) + 1
key = self._int2str(newpos)
if len(key) > self.steplen:
raise PathOverflow(_("Path Overflow from: '%s'" % (self.path, )))
return '%s%s%s' % (self.path[:-self.steplen], '0' *
(self.steplen - len(key)), key)
def _inc_path(cls, path):
""":returns: The path of the next sibling of a given node path."""
newpos = cls._str2int(path[-cls.steplen:]) + 1
key = cls._int2str(newpos)
if len(key) > cls.steplen:
raise PathOverflow(_("Path Overflow from: '%s'" % (path, )))
return '%s%s%s' % (path[:-cls.steplen], '0' *
(cls.steplen - len(key)), key)
def add_child(self, **kwargs):
"""
Adds a child to the node.
:raise PathOverflow: when no more child nodes can be added
"""
if not self.is_leaf() and self.node_order_by:
# there are child nodes and node_order_by has been set
# delegate sorted insertion to add_sibling
# we increase the numchild value of the object in memory, but can't
self.numchild += 1
return self.get_last_child().add_sibling('sorted-sibling',
**kwargs)
# creating a new object
newobj = self.__class__(**kwargs)
newobj.depth = self.depth + 1
if not self.is_leaf():
# adding the new child as the last one
newobj.path = self._inc_path(self.get_last_child().path)
else:
# the node had no children, adding the first child
newobj.path = self._get_path(self.path, newobj.depth, 1)
if len(newobj.path) > \
newobj.__class__._meta.get_field('path').max_length:
raise PathOverflow(
_('The new node is too deep in the tree, try'
' increasing the path.max_length property'
' and UPDATE your database'))
# saving the instance before returning it
newobj.save()
newobj._cached_parent_obj = self
# we increase the numchild value of the object in memory, but can't
# save because that makes this django 1.0 compatible code explode
self.numchild += 1
# we need to use a raw query
sql = "UPDATE %(table)s " \
"SET numchild=numchild+1 " \
"WHERE path=%%s" % {
'table': connection.ops.quote_name(
self.__class__._meta.db_table)}
cursor = connection.cursor()
cursor.execute(sql, [self.path])
transaction.commit_unless_managed()
return newobj
def process(self):
if self.node_cls.node_order_by and not self.node.is_leaf():
# there are child nodes and node_order_by has been set
# delegate sorted insertion to add_sibling
self.node.numchild += 1
return self.node.get_last_child().add_sibling(
'sorted-sibling', **self.kwargs)
if len(self.kwargs) == 1 and 'instance' in self.kwargs:
# adding the passed (unsaved) instance to the tree
newobj = self.kwargs['instance']
if newobj.pk:
raise NodeAlreadySaved("Attempted to add a tree node that is "\
"already in the database")
else:
# creating a new object
newobj = self.node_cls(**self.kwargs)
newobj.depth = self.node.depth + 1
if self.node.is_leaf():
# the node had no children, adding the first child
newobj.path = self.node_cls._get_path(
self.node.path, newobj.depth, 1)
max_length = self.node_cls._meta.get_field('path').max_length
if len(newobj.path) > max_length:
raise PathOverflow(
_('The new node is too deep in the tree, try'
' increasing the path.max_length property'
' and UPDATE your database'))
else:
# adding the new child as the last one
newobj.path = self.node.get_last_child()._inc_path()
get_result_class(self.node_cls).objects.filter(
path=self.node.path).update(numchild=F('numchild')+1)
# we increase the numchild value of the object in memory
self.node.numchild += 1
# saving the instance before returning it
newobj._cached_parent_obj = self.node
newobj.save()
return newobj
def add_child_bulk(self, parent, node_data):
# @@@ forked version of `Node._inc_path`
# https://github.com/django-treebeard/django-treebeard/blob/master/treebeard/mp_tree.py#L1121
child_node = Node(**node_data)
child_node.depth = parent.depth + 1
last_child = self.node_last_child_lookup.get(parent.urn)
if not last_child:
# The node had no children, adding the first child.
child_node.path = Node._get_path(parent.path, child_node.depth, 1)
if self.check_depth(child_node.path):
raise PathOverflow(
ugettext_noop("The new node is too deep in the tree, try"
" increasing the path.max_length property"
" and UPDATE your database"))
else:
# Adding the new child as the last one.
child_node.path = last_child._inc_path()
self.node_last_child_lookup[parent.urn] = child_node
self.nodes_to_create.append(child_node)
return child_node
def process(self):
if self.node.object_id != self.kwargs.get('object_id', False):
raise KeyError(
"The object_id for parent and child must be the same")
if self.node_cls.node_order_by and not self.node.is_leaf():
# there are child nodes and node_order_by has been set
# delegate sorted insertion to add_sibling
self.node.numchild += 1
return self.node.get_last_child().add_sibling(
'sorted-sibling', **self.kwargs)
# creating a new object
newobj = self.node_cls(**self.kwargs)
newobj.depth = self.node.depth + 1
if self.node.is_leaf():
# the node had no children, adding the first child
newobj.path = self.node_cls._get_path(self.node.path, newobj.depth,
1)
max_length = self.node_cls._meta.get_field('path').max_length
if len(newobj.path) > max_length:
raise PathOverflow(
_('The new node is too deep in the tree, try'
' increasing the path.max_length property'
' and UPDATE your database'))
else:
# adding the new child as the last one
newobj.path = self.node.get_last_child()._inc_path()
# saving the instance before returning it
newobj.save()
newobj._cached_parent_obj = self.node
self.node_cls.objects.filter(
path=self.node.path,
object_id=self.node.object_id).update(numchild=F('numchild') + 1)
# we increase the numchild value of the object in memory
self.node.numchild += 1
transaction.commit_unless_managed()
return newobj
def add_child(self, **kwargs):
"""
Adds a child to the node.
:raise PathOverflow: when no more child nodes can be added
"""
if not self.is_leaf() and self.node_order_by:
# there are child nodes and node_order_by has been set
# delegate sorted insertion to add_sibling
self.numchild += 1
return self.get_last_child().add_sibling('sorted-sibling',
**kwargs)
# creating a new object
newobj = self.__class__(**kwargs)
newobj.depth = self.depth + 1
if not self.is_leaf():
# adding the new child as the last one
newobj.path = self._inc_path(self.get_last_child().path)
else:
# the node had no children, adding the first child
newobj.path = self._get_path(self.path, newobj.depth, 1)
max_length = newobj.__class__._meta.get_field('path').max_length
if len(newobj.path) > max_length:
raise PathOverflow(
_('The new node is too deep in the tree, try'
' increasing the path.max_length property'
' and UPDATE your database'))
# saving the instance before returning it
newobj.save()
newobj._cached_parent_obj = self
self.__class__.objects.filter(path=self.path).update(
numchild=F('numchild') + 1)
# we increase the numchild value of the object in memory
self.numchild += 1
transaction.commit_unless_managed()
return newobj
def fix_tree(cls, fix_paths=False, **kwargs):
super().fix_tree(**kwargs)
if fix_paths:
with transaction.atomic():
# To fix holes and mis-orderings in paths, we consider each non-leaf node in turn
# and ensure that its children's path values are consecutive (and in the order
# given by node_order_by, if applicable). children_to_fix is a queue of child sets
# that we know about but have not yet fixed, expressed as a tuple of
# (parent_path, depth). Since we're updating paths as we go, we must take care to
# only add items to this list after the corresponding parent node has been fixed
# (and is thus not going to change).
# Initially children_to_fix is the set of root nodes, i.e. ones with a path
# starting with '' and depth 1.
children_to_fix = [('', 1)]
while children_to_fix:
parent_path, depth = children_to_fix.pop(0)
children = cls.objects.filter(path__startswith=parent_path,
depth=depth).values(
'pk', 'path', 'depth',
'numchild')
desired_sequence = children.order_by(
*(cls.node_order_by or ['path']))
# mapping of current path position (converted to numeric) to item
actual_sequence = {}
# highest numeric path position currently in use
max_position = None
# loop over items to populate actual_sequence and max_position
for item in desired_sequence:
actual_position = cls._str2int(
item['path'][-cls.steplen:])
actual_sequence[actual_position] = item
if max_position is None or actual_position > max_position:
max_position = actual_position
# loop over items to perform path adjustments
for (i, item) in enumerate(desired_sequence):
desired_position = i + 1 # positions are 1-indexed
actual_position = cls._str2int(
item['path'][-cls.steplen:])
if actual_position == desired_position:
pass
else:
# if a node is already in the desired position, move that node
# to max_position + 1 to get it out of the way
occupant = actual_sequence.get(desired_position)
if occupant:
old_path = occupant['path']
max_position += 1
new_path = cls._get_path(
parent_path, depth, max_position)
if len(new_path) > len(old_path):
previous_max_path = cls._get_path(
parent_path, depth, max_position - 1)
raise PathOverflow(
"Path Overflow from: '%s'" %
(previous_max_path, ))
cls._rewrite_node_path(old_path, new_path)
# update actual_sequence to reflect the new position
actual_sequence[max_position] = occupant
del (actual_sequence[desired_position])
occupant['path'] = new_path
# move item into the (now vacated) desired position
old_path = item['path']
new_path = cls._get_path(parent_path, depth,
desired_position)
cls._rewrite_node_path(old_path, new_path)
# update actual_sequence to reflect the new position
actual_sequence[desired_position] = item
del (actual_sequence[actual_position])
item['path'] = new_path
if item['numchild']:
# this item has children to process, and we have now moved the parent
# node into its final position, so it's safe to add to children_to_fix
children_to_fix.append((item['path'], depth + 1))
def fix_tree(cls, destructive=False, fix_paths=False):
"""
Solves some problems that can appear when transactions are not used and
a piece of code breaks, leaving the tree in an inconsistent state.
The problems this method solves are:
1. Nodes with an incorrect ``depth`` or ``numchild`` values due to
incorrect code and lack of database transactions.
2. "Holes" in the tree. This is normal if you move/delete nodes a
lot. Holes in a tree don't affect performance,
3. Incorrect ordering of nodes when ``node_order_by`` is enabled.
Ordering is enforced on *node insertion*, so if an attribute in
``node_order_by`` is modified after the node is inserted, the
tree ordering will be inconsistent.
:param fix_paths:
A boolean value. If True, a slower, more complex fix_tree method
will be attempted. If False (the default), it will use a safe (and
fast!) fix approach, but it will only solve the ``depth`` and
``numchild`` nodes, it won't fix the tree holes or broken path
ordering.
:param destructive:
Deprecated; alias for ``fix_paths``.
"""
cls = get_result_class(cls)
vendor = cls.get_database_vendor('write')
cursor = cls._get_database_cursor('write')
# fix the depth field
# we need the WHERE to speed up postgres
sql = (
"UPDATE %s "
"SET depth=" + sql_length("path", vendor=vendor) + "/%%s "
"WHERE depth!=" + sql_length("path", vendor=vendor) + "/%%s"
) % (connection.ops.quote_name(cls._meta.db_table), )
vals = [cls.steplen, cls.steplen]
cursor.execute(sql, vals)
# fix the numchild field
vals = ['_' * cls.steplen]
# the cake and sql portability are a lie
if cls.get_database_vendor('read') == 'mysql':
sql = (
"SELECT tbn1.path, tbn1.numchild, ("
"SELECT COUNT(1) "
"FROM %(table)s AS tbn2 "
"WHERE tbn2.path LIKE " +
sql_concat("tbn1.path", "%%s", vendor=vendor) + ") AS real_numchild "
"FROM %(table)s AS tbn1 "
"HAVING tbn1.numchild != real_numchild"
) % {'table': connection.ops.quote_name(cls._meta.db_table)}
else:
subquery = "(SELECT COUNT(1) FROM %(table)s AS tbn2"\
" WHERE tbn2.path LIKE " + sql_concat("tbn1.path", "%%s", vendor=vendor) + ")"
sql = ("SELECT tbn1.path, tbn1.numchild, " + subquery +
" FROM %(table)s AS tbn1 WHERE tbn1.numchild != " +
subquery)
sql = sql % {
'table': connection.ops.quote_name(cls._meta.db_table)}
# we include the subquery twice
vals *= 2
cursor.execute(sql, vals)
sql = "UPDATE %(table)s "\
"SET numchild=%%s "\
"WHERE path=%%s" % {
'table': connection.ops.quote_name(cls._meta.db_table)}
for node_data in cursor.fetchall():
vals = [node_data[2], node_data[0]]
cursor.execute(sql, vals)
if fix_paths or destructive:
with transaction.atomic():
# To fix holes and mis-orderings in paths, we consider each non-leaf node in turn
# and ensure that its children's path values are consecutive (and in the order
# given by node_order_by, if applicable). children_to_fix is a queue of child sets
# that we know about but have not yet fixed, expressed as a tuple of
# (parent_path, depth). Since we're updating paths as we go, we must take care to
# only add items to this list after the corresponding parent node has been fixed
# (and is thus not going to change).
# Initially children_to_fix is the set of root nodes, i.e. ones with a path
# starting with '' and depth 1.
children_to_fix = [('', 1)]
while children_to_fix:
parent_path, depth = children_to_fix.pop(0)
children = cls.objects.filter(
path__startswith=parent_path, depth=depth
).values('pk', 'path', 'depth', 'numchild')
desired_sequence = children.order_by(*(cls.node_order_by or ['path']))
# mapping of current path position (converted to numeric) to item
actual_sequence = {}
# highest numeric path position currently in use
max_position = None
# loop over items to populate actual_sequence and max_position
for item in desired_sequence:
actual_position = cls._str2int(item['path'][-cls.steplen:])
actual_sequence[actual_position] = item
if max_position is None or actual_position > max_position:
max_position = actual_position
# loop over items to perform path adjustments
for (i, item) in enumerate(desired_sequence):
desired_position = i + 1 # positions are 1-indexed
actual_position = cls._str2int(item['path'][-cls.steplen:])
if actual_position == desired_position:
pass
else:
# if a node is already in the desired position, move that node
# to max_position + 1 to get it out of the way
occupant = actual_sequence.get(desired_position)
if occupant:
old_path = occupant['path']
max_position += 1
new_path = cls._get_path(parent_path, depth, max_position)
if len(new_path) > len(old_path):
previous_max_path = cls._get_path(parent_path, depth, max_position - 1)
raise PathOverflow(_("Path Overflow from: '%s'" % (previous_max_path, )))
cls._rewrite_node_path(old_path, new_path)
# update actual_sequence to reflect the new position
actual_sequence[max_position] = occupant
del(actual_sequence[desired_position])
occupant['path'] = new_path
# move item into the (now vacated) desired position
old_path = item['path']
new_path = cls._get_path(parent_path, depth, desired_position)
cls._rewrite_node_path(old_path, new_path)
# update actual_sequence to reflect the new position
actual_sequence[desired_position] = item
del(actual_sequence[actual_position])
item['path'] = new_path
if item['numchild']:
# this item has children to process, and we have now moved the parent
# node into its final position, so it's safe to add to children_to_fix
children_to_fix.append((item['path'], depth + 1))