def fix_edges(edges):
# turn all "fake" (negative) verticies monotonically increasing numbers...
# TODO: this could probably be made more efficient by doing a topo sort
# directly, rather than using this (helpful) library
verticies = set([])
pairs = []
for e in edges:
verticies.add(e.span[0])
verticies.add(e.span[1])
pairs.append(e.span)
sorted_verticies = toposort.topological_sort(verticies, pairs)
if sorted_verticies == None:
sys.stderr.write("cannot process lattice -- could there be a loop?\n")
return [lattice.Vertex(id=0, label='error_in_lattice_toposort')]
vertex_transform = {}
for i in xrange(len(verticies)):
vertex_transform[sorted_verticies[i]] = i
return [
lattice.Edge(span=(vertex_transform[e.span[0]],
vertex_transform[e.span[1]]),
label=e.label,
properties=e.properties) for e in edges
]
def toXmlDB(self):
uids = sorted(self.umbrellas.keys())
cids = sorted(self.collectors.keys())
# topological sort of the alert
alerts_sorted = topological_sort(self.alerts.values(),
lambda a: a.dependson)
s = '<alertdb>\n'
s += ' <alerts>\n'
for alert in alerts_sorted:
if not alert.skip:
s += alert.toXml()
for uid in uids:
s += self.toXmlUmbrella(uid, self.umbrellas[uid])
for cid in cids:
s += self.toXmlCollector(cid, self.collectors[cid])
s += ' </alerts>\n'
s += ' <parameters>\n'
for key in self.paramSorted:
s += self.parameters[key].toXmlIntern()
s += ' </parameters>\n'
s += '</alertdb>\n'
return s
def fix_edges(edges):
# turn all "fake" (negative) verticies monotonically increasing numbers...
# TODO: this could probably be made more efficient by doing a topo sort
# directly, rather than using this (helpful) library
verticies = set([])
pairs = []
for e in edges:
verticies.add(e.span[0])
verticies.add(e.span[1])
pairs.append(e.span)
sorted_verticies = toposort.topological_sort(verticies, pairs)
if sorted_verticies == None:
sys.stderr.write("cannot process lattice -- could there be a loop?\n")
return [lattice.Vertex(id=0, label='error_in_lattice_toposort')]
vertex_transform = {}
for i in xrange(len(verticies)):
vertex_transform[sorted_verticies[i]] = i
return [lattice.Edge(span=(vertex_transform[e.span[0]],vertex_transform[e.span[1]]),label=e.label,properties=e.properties) for e in edges]
def render(self):
inclusions = topological_sort(
self._inclusions, lambda inclusion: inclusion.dependencies())
snippets = [inclusion.html() for inclusion in inclusions]
return '\n'.join(snippets)
def derive_block_info(dbfile):
"""Store for each block to which account chain (account id) it belongs"""
print('Deriving per-block info')
sqldb = apsw.Connection(dbfile)
sqlcur = sqldb.cursor()
sqlcur.execute('delete from block_info')
# Previous point for a block, or None (for open blocks)
# Key: block id
# Value: id of previous block
block_to_previous = {}
# Get open blocks, for which we know the account
open_block_to_account = {}
account_to_open_block = {}
block_to_type = {}
block_to_sister = {}
block_to_balance = {}
sqlcur.execute('select id, account, source from blocks where type=?',
('open', ))
for id, account, source in sqlcur:
block_to_previous[id] = None
block_to_type[id] = 'open'
open_block_to_account[id] = account
account_to_open_block[account] = id
if id == 0:
# No source for genesis open block
continue
assert source is not None
block_to_sister[id] = source
block_to_sister[source] = id
# Gather all other blocks
blocks_to_process = set()
sqlcur.execute(
'select id, type, previous, source, destination, balance_raw from blocks where type<>?',
('open', ))
for id, type, previous, source, destination, balance in sqlcur:
block_to_type[id] = type
if previous is None:
print('No previous value for block %d (type %s)!' % (id, type))
if type == 'receive':
assert source is not None
block_to_sister[id] = source
block_to_sister[source] = id
elif type == 'send':
block_to_balance[id] = int(balance)
block_to_previous[id] = previous
blocks_to_process.add(id)
# Reconstruct all the account chains, using the previous pointers
# in the blocks
# Account chains under construction
# Key: id of *last* block in the chain
# Value: list of sequential blocks [open, ... , D, E, ...]; with E.previous = D
account_chains = {}
# Bootstrap with the open blocks
for id in open_block_to_account.keys():
account_chains[id] = [id]
bar = progressbar.ProgressBar('Reconstructing account chains')
bar.update(len(blocks_to_process))
while len(blocks_to_process) > 0:
# Get next block to process
tail_block = blocks_to_process.pop()
#print('processing block %d' % tail_block)
# Follow the previous pointers until we hit an existing chain
chain = collections.deque([tail_block])
blocks_processed = set()
cur_block = block_to_previous[tail_block]
while cur_block not in account_chains:
assert cur_block is not None
assert cur_block not in blocks_processed
chain.appendleft(cur_block)
blocks_processed.add(cur_block)
cur_block = block_to_previous[cur_block]
assert cur_block in account_chains
#print('processed block %d: merging chain of %d with %d' % (tail_block, len(account_chains[cur_block]), len(chain)))
# Merge the new chain with the existing one
new_chain = account_chains[cur_block] + list(chain)
del account_chains[cur_block]
account_chains[new_chain[-1]] = new_chain
blocks_to_process -= blocks_processed
bar.update(len(blocks_to_process))
bar.finish(len(account_chains))
assert len(account_chains) == len(open_block_to_account)
# Determine block -> account mapping
block_to_account = {}
for last_block, chain in account_chains.items():
account = open_block_to_account[chain[0]]
for block in chain:
block_to_account[block] = account
# Compute account balance at each block, plus amounts transfered
# by send/receive/open blocks.
block_to_amount = {}
compute_block_balances_and_amounts(account_chains, block_to_type,
block_to_previous, block_to_sister,
block_to_balance, block_to_amount)
# Perform global topological sort of all blocks, based on
# dependencies between blocks
edges = generate_block_dependencies(sqlcur, account_to_open_block,
block_to_account)
print('Determining topological order')
order = topological_sort(edges)
block_to_global_index = {}
for idx, block in enumerate(order):
block_to_global_index[block] = idx
# Store all the derived information
bar = progressbar.ProgressBar('Storing per-block info for each account')
bar.update(len(blocks_to_process))
i = 0
sqlcur.execute('begin')
for last_block, chain in account_chains.items():
account = open_block_to_account[chain[0]]
for idx, block in enumerate(chain):
sister = None
if block in block_to_sister:
sister = block_to_sister[block]
# Long int -> string
balance = str(block_to_balance[block])
amount = None
if block in block_to_amount:
amount = str(block_to_amount[block])
sqlcur.execute(
'insert into block_info (block, account, chain_index, global_index, sister, balance, amount) values (?,?,?,?,?,?,?)',
(block, account, idx, block_to_global_index[block], sister,
balance, amount))
i += 1
bar.update(i)
sqlcur.execute('commit')
bar.finish()
