def checkchildren(self, level, index, locked=False):
"""
Recursively checks the descendants of a node to see if they can be
validated.
"""
# fixme: there's a lot of duplicate code between this and addhash
if not locked: self.rwlock.acquire_write()
key = (level, index)
hash = self.getnode(level, index, locked=True)
assert hash
assert not key in self.purgatory
c1k, c2k = ((level + 1, index * 2), (level + 1, index * 2 + 1)
) # child 1/2 key
keys = [key, c1k, c2k]
hashes = [hash, None, None]
for i in range(1, 3):
k = keys[i]
hashes[i] = self.purgatory[k] if k in self.purgatory else None
if not hashes[i]:
if not keys[i][
1] & 1: # if even (left sibling), we check to see if this is the right edge of the tree
if self.txcount:
height = int(math.ceil(math.log(self.txcount, 2)))
if keys[i][0] > height: continue
edge = (self.txcount - 1) >> (height - keys[i][0])
if index * 2 == edge:
#print "found edge at ", keys[i]
hashes[i] = hashes[
1] # this can be overwritten later
break
if not hashes[1] or not hashes[2]:
# One of the child keys is not available; abort validation.
# This can occur if intermediate hashes have been added
# via. addhash(), and not all descendants of that
# intermediate hash have been added.
if not locked: self.rwlock.release_write()
return
if self.calcparent(hashes[1], hashes[2]) == hashes[0]:
if hashes[1] == hashes[
2] and level == self.levels: # right edge, bottom row
self.txcount = index | 1 # left sib is the last tx, but we start counting from 0, so we want the right sib's index
is_edge = hashes[1] == hashes[2]
for i in range(1, 3):
self.setnode(keys[i][0],
keys[i][1],
hashes[i],
edge=is_edge,
locked=True)
del self.purgatory[keys[i]]
if not is_edge or i < 3:
self.checkchildren(keys[i][0], keys[i][1], locked=True)
else:
debuglog(
'bttree',
"Invalid descendants encountered in checkchildren. This should not happen. Keys: ",
keys)
if not locked: self.rwlock.release_write()
def checkchildren(self, level, index):
"""
Recursively checks the descendents of a node to see if they can be
validated.
"""
# fixme: there's a lot of duplicate code between this and addhash
key = (level, index)
hash = self.getnode(level, index)
assert hash
assert not key in self.purgatory
c1k, c2k = ((level + 1, index * 2), (level + 1, index * 2 + 1)
) # child 1/2 key
keys = [key, c1k, c2k]
hashes = [hash, None, None]
for i in range(1, 3):
k = keys[i]
hashes[i] = self.purgatory[k] if k in self.purgatory else None
if not hashes[i]:
if not keys[i][
1] & 1: # if even (left sibling), we check to see if this is the right edge of the tree
for hint in self.txcounthints:
height = int(math.ceil(math.log(hint, 2)))
if keys[i][0] > height: continue
edge = (hint - 1) >> (height - keys[i][0])
if index * 2 == edge:
print "found edge at ", keys[[i]]
hashes[i] = hashes[
1] # this can be overwritten later
break
else:
if level <= int(
math.ceil(math.log(max(self.txcounthints),
2))) and index <= max(
self.txcounthints):
debuglog(
'bttree',
"Couldn't find hash for %i %i when checking %i %i"
% (k[0], k[1], key[0], key[1]))
return
if self.calcparent(hashes[1], hashes[2]) == hashes[0]:
if hashes[1] == hashes[
2] and level == self.levels: # right edge, bottom row
self.txcount = index | 1 - 1 # left sib's index
for i in range(1, 3):
self.setnode(keys[i][0],
keys[i][1],
hashes[i],
edge=(hashes[1] == hashes[2]))
del self.purgatory[keys[i]]
self.checkchildren(keys[i][0], keys[i][1])
else:
debuglog(
'bttree',
"Invalid descendents encountered in checkchildren. This should not happen. Keys: ",
keys)
def checkchildren(self, level, index):
"""
Recursively checks the descendents of a node to see if they can be
validated.
"""
# fixme: there's a lot of duplicate code between this and addhash
key = (level, index)
hash = self.getnode(level, index)
assert hash
assert not key in self.purgatory
c1k, c2k = ((level+1, index*2), (level+1, index*2+1)) # child 1/2 key
keys = [key, c1k, c2k]
hashes = [hash, None, None]
for i in range(1, 3):
k = keys[i]
hashes[i] = self.purgatory[k] if k in self.purgatory else None
if not hashes[i]:
if not keys[i][1] & 1: # if even (left sibling), we check to see if this is the right edge of the tree
for hint in self.txcounthints:
height = int(math.ceil(math.log(hint, 2)))
if keys[i][0] > height: continue
edge = (hint-1) >> (height - keys[i][0])
if index*2 == edge:
print "found edge at ", keys[[i]]
hashes[i] = hashes[1] # this can be overwritten later
break
else:
if level <= int(math.ceil(math.log(max(self.txcounthints), 2))) and index <= max(self.txcounthints):
debuglog('bttree', "Couldn't find hash for %i %i when checking %i %i" % (k[0], k[1], key[0], key[1]))
return
if self.calcparent(hashes[1], hashes[2]) == hashes[0]:
if hashes[1] == hashes[2] and level == self.levels: # right edge, bottom row
self.txcount = index|1-1 # left sib's index
for i in range(1,3):
self.setnode(keys[i][0], keys[i][1], hashes[i], edge=(hashes[1]==hashes[2]))
del self.purgatory[keys[i]]
self.checkchildren(keys[i][0], keys[i][1])
else:
debuglog('bttree', "Invalid descendents encountered in checkchildren. This should not happen. Keys: ", keys)
def setnode(self, level, index, hash, edge=False):
"""
Sets the hash at the specified level and index of the validated tree.
The parent hash must have already been added.
"""
assert index < 2**level
assert level >= 0
assert level <= config.MAX_DEPTH
i = index # of subtree
L = level # of subtree
s = self.valid # the subtree
while 1:
if L==0: # this node is the target
s.extend([hash, [], []])
self.upgradestate(level, index, edge)
return
elif len(s) < 3:
debuglog('bttree', 'setnode(%i, %i, hash) found undersized element at %i, %i: %s' % (level, index, L, i, `s`))
raise
L -= 1
s = s[1 + ((i>>L)%2)] # take the left or right subtree
i = i % (1<<L) # we just took that step; clear the bit for sanity's sake
def addhash(self, level, index, hash, peer=None, locked=False):
"""
Adds a hash to either the validated tree (when possible) or to the
unvalidated cache, self.purgatory. This will also add any computed parent
hashes recursively. If the hash makes it into the validated tree, this
will also check the nephews of this hash to see if they can now be
validated. However, direct descendants will not be checked, and must be
checked by the caller.
"""
if type(hash) == long:
hash = util.ser_uint256(hash)
key = (level, index)
if not locked:
self.rwlock.acquire_write(
) # this might benefit from more fine-grained locks when the switch is made to C++
if key in self.purgatory:
if self.purgatory[key] == hash:
if not locked: self.rwlock.release_write()
return
else:
oldpeer = self.peerorigins[self.purgatory[
key]] if self.purgatory[key] in self.peerorigins else None
debuglog(
'btnet',
'Warning: received two different hashes for the same part of a tree. Replacing old hash.'
)
debuglog(
'btnet',
'Cause is likely either network corruption or a malicious peer. Peers:'
)
debuglog('btnet', oldpeer, peer)
debuglog(
'btnet', 'Hash added is (%i, %i): %s. Oldhash: %s.' %
(level, index, to_hex(hash), to_hex(self.purgatory[key])))
# fixme: peer banning
# continue to add the new hash and validate
elif self.getnode(level, index, locked=True):
debuglog(
'bttree',
'Debug warning: level=%i index=%i already validated in tree' %
(level, index))
if not locked: self.rwlock.release_write()
return
self.purgatory[key] = hash
#self.peerorigins[hash] = peer # fixme: make sure memory growth is bounded
parent = self.getnode(level - 1, index // 2,
locked=True) # is our parent already valid?
siblingkey = (level, index ^ 1)
if not siblingkey in self.purgatory: # Is this is the right edge of the tree?
if not index & 1: # if even (left sibling)
if self.txcount:
height = int(math.ceil(math.log(self.txcount, 2)))
assert level <= height
edge = (self.txcount - 1) >> (height - level)
if index == edge:
self.purgatory[
siblingkey] = hash # this can be overwritten later
if siblingkey in self.purgatory: # then we can check one level up
sib = self.purgatory[siblingkey]
parenthash = self.calcparent(sib, hash) if (
index %
2) else self.calcparent(hash, sib) # left sibling goes first
if parent and parent == parenthash:
result = 'connected'
elif parent and parent != parenthash and not sib == hash:
debuglog(
'btnet',
'Invalid hash(es) encountered when checking (%i, %i): %s.'
% (level, index, to_hex(hash)))
debuglog(
'btnet', 'Parent (%i, %i) = %s not %s' %
(level - 1, index // 2, to_hex(parent),
to_hex(parenthash)))
result = 'invalid'
elif parent and parent != parenthash and sib == hash:
debuglog(
'btnet', 'Found a bad edge: (%i, %i) = %s not %s' %
(level - 1, index // 2, to_hex(parent),
to_hex(parenthash)))
result = 'orphan' # incorrect tx count hint
else: # recurse one level up
result = self.addhash(level - 1,
index // 2,
parenthash,
None,
locked=True)
else:
result = 'orphan'
if result == 'connected':
self.setnode(level, index, hash, edge=(hash == sib), locked=True)
self.setnode(level,
index ^ 1,
sib,
edge=(hash == sib),
locked=True)
del self.purgatory[key]
del self.purgatory[siblingkey]
if hash == sib and level == self.levels: # right edge, bottom row
self.txcount = index | 1 # left sib is the last tx, but we start counting from 0, so we want the right sib's index
# the recursive caller of addhash will take care of the children of key, but not siblingkey
if hash != sib:
self.checkchildren(siblingkey[0], siblingkey[1], locked=True)
elif result == 'invalid':
if sib == hash: # invalid hint about the number of transactions
debuglog('btnet', 'Invalid txcount? -- %i ' % self.txcount)
del self.purgatory[max(siblingkey, key)]
result = 'orphan'
else:
for k in key, siblingkey:
# fixme: for multi-level recursion, there's a good chance we're deleting the wrong txes.
# should we delete all of the descendants of the lowest valid hash to which this resolves?
# or should we leave these hashes all in purgatory? or what? who do we ban?
debuglog(
'btnet',
'Invalid hash(es) encountered. Deleting: (%i, %i): %s.'
% (k[0], k[1], to_hex(self.purgatory[k])))
#del self.purgatory[k]
elif result == 'orphan':
pass # fixme: deal with peer info (and banning) in each of these branches above
if not locked: self.rwlock.release_write()
return result
def setnode(self, level, index, value, locked=False):
"""
Sets the state of a node of the tree, specified by its level, index, and
new value. Creates and destroys nodes as needed to ensure that the TreeState
node population rules are preserved. For example, setting an internal node to
a value of 2 will remove all its descendants from the tree (even if they have
a value of 3 -- careful!).
"""
assert index < 2**level
assert level >= 0
assert level <= config.MAX_DEPTH
assert value in (0, 1, 2, 3)
if not locked: self.rwlock.acquire_write()
if value == 0:
raise NotImplementedError # clearing inventory is not supported
# Algorithm: we walk down the tree until we get to the target,
# creating nodes as needed to get to the target, then we walk back
# up and clear any nodes that were made redundant by the changes we just made
# "Down" means away from the root (towards the children)
i = index # of subtree
L = level # of subtree
s = self.state # the subtree
ancestors = [] # t
while L > 0:
v = s[0]
if v > value: # this can probably happen from out-of-order packets. Remove later.
debuglog(
'bttree',
'Debug warning: Parent is more complete than decendants %i %i'
% (L, i))
if not locked: self.rwlock.release_write()
return
elif v == value and v != 1:
break
elif v in (0, 2, 3) and v != value:
# this node has no children. Let's add them, being careful to mutate
# the list instead of replacing it in order to ensure that we're modifying
# the actual tree and not a copied subtree
assert len(s) == 1
s[0] = 1
s.extend([[v], [v]]) # accidental code emoji
ancestors.append(s)
L -= 1
s = s[1 + ((i >> L) % 2)] # take the left or right subtree
i = i % (
1 << L
) # we just took that step; clear the bit for sanity's sake
if L == 0:
v = s[0]
if v == value:
if not locked: self.rwlock.release_write()
return # nothing to see here, move along
if v > value: # this can probably happen from out-of-order packets. Remove later.
if not locked: self.rwlock.release_write()
return
self.changes += 1
if value == 1:
assert len(s) == 1
assert s[0] <= value
s[0] = 1
s.extend([[0], [0]])
else: # value == 2 or 3
del s[:]
s.append(value)
ancestors.append(s)
# now let's go through the ancestors and remove redundancies
while ancestors:
s = ancestors.pop()
if s[0] in (0, 2, 3): continue
left, right = s[1][0], s[2][0]
if left == right and (left > 1):
del s[:]
s.append(left)
if not locked: self.rwlock.release_write()
return
def addhash(self, level, index, hash, peer=None):
"""
Adds a hash to either the validated tree (when possible) or to the
unvalidated cache, self.purgatory. This will also add any computed parent
hashes recursively. If the hash makes it into the validated tree, this
will also check the nephews of this hash to see if they can now be
validated. However, direct descendents will not be checked, and must be
checked by the caller.
"""
if type(hash) == long:
hash = util.ser_uint256(hash)
key = (level, index)
if key in self.purgatory:
if self.purgatory[key] == hash:
return
else:
oldpeer = self.peerorigins[self.purgatory[key]] if self.purgatory[key] in self.peerorigins else None
debuglog('btnet', 'Warning: received two different hashes for the same part of a tree. Replacing old hash.')
debuglog('btnet', 'Cause is likely either network corruption or a malicious peer. Peers:')
debuglog('btnet', oldpeer, peer)
debuglog('btnet', 'Hash added is (%i, %i): %s. Oldhash: %s.' % (level, index, to_hex(hash), to_hex(self.purgatory[key])))
# fixme: peer banning
# continue to add the new hash and validate
elif self.getnode(level, index):
debuglog('bttree', 'Debug warning: level=%i index=%i already validated in tree' % (level, index))
return
self.purgatory[key] = hash
#self.peerorigins[hash] = peer # fixme: make sure memory growth is bounded
parent = self.getnode(level-1, index//2) # is our parent already valid?
#if parent: print "valid parent of %i,%i is %i,%i:" %(level, index, level-1, index//2), to_hex(parent])
siblingkey = (level, index ^ 1)
if not siblingkey in self.purgatory: # Is this is the right edge of the tree?
if not index & 1: # if even (left sibling)
for hint in self.txcounthints:
height = int(math.ceil(math.log(hint, 2)))
if level > height: continue
edge = (hint-1) >> (height - level)
if index == edge:
self.purgatory[siblingkey] = hash # this can be overwritten later
break
if siblingkey in self.purgatory: # then we can check one level up
sib = self.purgatory[siblingkey]
parenthash = self.calcparent(sib, hash) if (index%2) else self.calcparent(hash, sib) # left sibling goes first
if parent and parent == parenthash:
result = 'connected'
elif parent and parent != parenthash and not sib == hash:
debuglog('btnet', 'Invalid hash(es) encountered when checking (%i, %i): %s.' % (level, index, to_hex(hash)))
debuglog('btnet', 'Parent (%i, %i) = %s not %s' % (level-1, index//2, to_hex(parent), to_hex(parenthash)))
result = 'invalid'
elif parent and parent != parenthash and sib == hash:
debuglog('btnet', 'Found a bad edge: (%i, %i) = %s not %s' % (level-1, index//2, to_hex(parent), to_hex(parenthash)))
result = 'orphan' # incorrect tx count hint
else: # recurse one level up
result = self.addhash(level-1, index//2, parenthash, None)
else:
result = 'orphan'
if result == 'connected':
self.setnode(level, index, hash, edge=(hash==sib))
self.setnode(level, index^1, sib, edge=(hash==sib))
del self.purgatory[key]
del self.purgatory[siblingkey]
if hash == sib and level == self.levels: # right edge, bottom row
self.txcount = index|1-1 # left sib's index
# the recursive caller of addhash will take care of the children of key, but not siblingkey
self.checkchildren(siblingkey[0], siblingkey[1])
elif result == 'invalid':
if sib == hash: # invalid hint about the number of transactions
debuglog('btnet', 'Invalid txcount hint: %i among ' % hint, self.txcounthints)
del self.purgatory[max(siblingkey, key)]
result = 'orphan'
else:
for k in key, siblingkey:
# fixme: for multi-level recursion, there's a good chance we're deleting the wrong txes.
# should we delete all of the decendants of the lowest valid hash to which this resolves?
# or should we leave these hashes all in purgatory? or what? who do we ban?
debuglog('btnet', 'Invalid hash(es) encountered. Deleting: (%i, %i): %s.' % (k[0], k[1], to_hex(self.purgatory[k])))
#del self.purgatory[k]
elif result == 'orphan':
pass # fixme: deal with peer info (and banning) in each of these branches above
return result
def setnode(self, level, index, value):
"""
Sets the state of a node of the tree, specified by its level, index, and
new value. Creates and destroys nodes as needed to ensure that the TreeState
node population rules are preserved. For example, setting an internal node to
a value of 2 will remove all its decendants from the tree (even if they have
a value of 3 -- careful!).
"""
assert index < 2**level
assert level >= 0
assert level <= config.MAX_DEPTH
assert value in (0,1,2,3)
if value == 0:
raise NotImplementedError # clearing inventory is not supported
# Algorithm: we walk down the tree until we get to the target,
# creating nodes as needed to get to the target, then we walk back
# up and clear any nodes that were made redundant by the changes we just made
# "Down" means away from the root (towards the children)
i = index # of subtree
L = level # of subtree
s = self.state # the subtree
ancestors = [] # t
while L > 0:
v = s[0]
if v > value: # this can probably happen from out-of-order packets. Remove later.
debuglog('bttree', 'Debug warning: Parent is more complete than decendants')
return
elif v == value and v != 1:
break
elif v in (0, 2, 3) and v != value:
# this node has no children. Let's add them, being careful to mutate
# the list instead of replacing it in order to ensure that we're modifying
# the actual tree and not a copied subtree
assert len(s) == 1
s[0] = 1
s.extend([[v],[v]]) # accidental code emoji
ancestors.append(s)
L -= 1
s = s[1 + ((i>>L)%2)] # take the left or right subtree
i = i % (1<<L) # we just took that step; clear the bit for sanity's sake
if L == 0:
v = s[0]
if v == value:
return # nothing to see here, move along
if v > value: # this can probably happen from out-of-order packets. Remove later.
return
if value == 1:
assert len(s) == 1
assert s[0] <= value
s[0] = 1
s.extend([[0],[0]])
else: # value == 2 or 3
del s[:]
s.append(value)
ancestors.append(s)
# now let's go through the ancestors and remove redundancies
while ancestors:
s = ancestors.pop()
if s[0] in (0, 2, 3): continue
left, right = s[1][0], s[2][0]
if left == right and (left > 1):
del s[:]
s.append(left)
return