def __init__(self, q_new_transaction):
"""
Initialize the class with the needed
"""
self._q_new_transaction = q_new_transaction
self.q_new_transaction = None
self.q_tx_to_validate = mp.Queue()
self.q_tx_validated = mp.Queue()
self.q_tx_delete = mp.Queue()
self.q_block = mp.Queue()
self.initialized = mp.Event()
self.monitor = Monitor()
def __init__(self, q_new_block):
"""
Initialize the class with the needed queues.
Initialize with a queue where new blocks added to the bigchain will be put
"""
self.monitor = Monitor()
self.q_new_block = q_new_block
self.q_blocks_to_validate = mp.Queue()
self.q_validated_block = mp.Queue()
self.q_voted_block = mp.Queue()
self.v_previous_block_id = mp.Value(ctypes.c_char_p)
self.initialized = mp.Event()
import logging
import multiprocessing as mp
import queue
import rethinkdb as r
import bigchaindb
from bigchaindb import Bigchain
from bigchaindb.monitor import Monitor
logger = logging.getLogger(__name__)
monitor = Monitor()
class Block(object):
def __init__(self, q_new_transaction):
"""
Initialize the class with the needed
"""
self._q_new_transaction = q_new_transaction
self.q_new_transaction = None
self.q_tx_to_validate = mp.Queue()
self.q_tx_validated = mp.Queue()
self.q_tx_delete = mp.Queue()
self.q_block = mp.Queue()
self.initialized = mp.Event()
def filter_by_assignee(self):
"""
Handle transactions that are assigned to me
class Voter(object):
def __init__(self, q_new_block):
"""
Initialize the class with the needed queues.
Initialize with a queue where new blocks added to the bigchain will be put
"""
self.monitor = Monitor()
self.q_new_block = q_new_block
self.q_blocks_to_validate = mp.Queue()
self.q_validated_block = mp.Queue()
self.q_voted_block = mp.Queue()
self.v_previous_block_id = mp.Value(ctypes.c_char_p)
self.v_previous_block_number = mp.Value(ctypes.c_uint64)
self.initialized = mp.Event()
def feed_blocks(self):
"""
Prepare the queue with blocks to validate
"""
block_stream = BlockStream(self.q_new_block)
while True:
# poison pill
block = block_stream.get()
if block == 'stop':
self.q_blocks_to_validate.put('stop')
return
self.q_blocks_to_validate.put(block)
def validate(self):
"""
Checks if incoming blocks are valid or not
"""
# create a bigchain instance. All processes should create their own bigchcain instance so that they all
# have their own connection to the database
b = Bigchain()
logger.info('voter waiting for new blocks')
# signal initialization complete
self.initialized.set()
while True:
new_block = self.q_blocks_to_validate.get()
# poison pill
if new_block == 'stop':
self.q_validated_block.put('stop')
return
logger.info('new_block arrived to voter')
block_number = self.v_previous_block_number.value + 1
with self.monitor.timer('validate_block'):
validity = b.is_valid_block(new_block)
self.q_validated_block.put(
(new_block, self.v_previous_block_id.value.decode(),
block_number, validity))
self.v_previous_block_id.value = new_block['id'].encode()
self.v_previous_block_number.value = block_number
def vote(self):
"""
Votes on the block based on the decision of the validation
"""
# create a bigchain instance
b = Bigchain()
while True:
elem = self.q_validated_block.get()
# poison pill
if elem == 'stop':
self.q_voted_block.put('stop')
return
validated_block, previous_block_id, block_number, decision = elem
vote = b.vote(validated_block, previous_block_id, decision)
self.q_voted_block.put((validated_block, vote, block_number))
def update_block(self):
"""
Appends the vote in the bigchain table
"""
# create a bigchain instance
b = Bigchain()
while True:
elem = self.q_voted_block.get()
# poison pill
if elem == 'stop':
logger.info('clean exit')
return
block, vote, block_number = elem
logger.info('updating block %s with number %s and with vote %s',
block['id'], block_number, vote)
b.write_vote(block, vote, block_number)
def bootstrap(self):
"""
Before starting handling the new blocks received by the changefeed we need to handle unvoted blocks
added to the bigchain while the process was down
We also need to set the previous_block_id and the previous block_number
"""
b = Bigchain()
last_voted = b.get_last_voted_block()
self.v_previous_block_number.value = last_voted['block_number']
self.v_previous_block_id.value = last_voted['id'].encode()
def kill(self):
"""
Terminate processes
"""
self.q_new_block.put('stop')
def start(self):
"""
Initialize, spawn, and start the processes
"""
self.bootstrap()
# initialize the processes
p_feed_blocks = mp.Process(name='block_feeder',
target=self.feed_blocks)
p_validate = mp.Process(name='block_validator', target=self.validate)
p_vote = mp.Process(name='block_voter', target=self.vote)
p_update = mp.Process(name='block_updater', target=self.update_block)
# start the processes
p_feed_blocks.start()
p_validate.start()
p_vote.start()
p_update.start()
class Block(object):
def __init__(self, q_new_transaction):
self._q_new_transaction = q_new_transaction
self.q_new_transaction = None
self.q_tx_to_validate = mp.Queue()
self.q_tx_validated = mp.Queue()
self.q_tx_delete = mp.Queue()
self.q_block = mp.Queue()
self.initialized = mp.Event()
self.monitor = Monitor()
def filter_by_assignee(self):
"""
Handle transactions that are assigned to me
"""
# create a bigchain instance
b = Bigchain()
while True:
tx = self.q_new_transaction.get()
# poison pill
if tx == 'stop':
self.q_tx_to_validate.put('stop')
return
if tx['assignee'] == b.me:
tx.pop('assignee')
self.q_tx_to_validate.put(tx)
def validate_transactions(self):
"""
Checks if the incoming transactions are valid
"""
# create a bigchain instance
b = Bigchain()
while True:
self.monitor.gauge('tx_queue_gauge',
self.q_tx_to_validate.qsize(),
rate=bigchaindb.config['statsd']['rate'])
tx = self.q_tx_to_validate.get()
# poison pill
if tx == 'stop':
self.q_tx_delete.put('stop')
self.q_tx_validated.put('stop')
return
self.q_tx_delete.put(tx['_id'])
with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']):
is_valid_transaction = b.is_valid_transaction(tx)
if is_valid_transaction:
self.q_tx_validated.put(tx)
def create_blocks(self):
"""
Create a block with valid transactions
"""
# create a bigchain instance
b = Bigchain()
stop = False
while True:
# read up to 1000 transactions
validated_transactions = []
for i in range(1000):
try:
tx = self.q_tx_validated.get(timeout=5)
except queue.Empty:
break
# poison pill
if tx == 'stop':
stop = True
break
validated_transactions.append(tx)
# if there are no transactions skip block creation
if validated_transactions:
# create block
block = b.create_block(validated_transactions)
self.q_block.put(block)
if stop:
self.q_block.put('stop')
return
def write_blocks(self):
"""
Write blocks to the bigchain
"""
# create bigchain instance
b = Bigchain()
# Write blocks
while True:
block = self.q_block.get()
# poison pill
if block == 'stop':
return
with self.monitor.timer('write_block'):
b.write_block(block)
def delete_transactions(self):
"""
Delete transactions from the backlog
"""
# create bigchain instance
b = Bigchain()
client=pymongo.MongoClient()
backlog=client.bigchain.backlog
stop = False
while True:
# try to delete in batch to reduce io
tx_to_delete = []
for i in range(1000):
try:
tx = self.q_tx_delete.get(timeout=5)
except queue.Empty:
break
# poison pill
if tx == 'stop':
stop = True
break
tx_to_delete.append(tx)
if len(tx_to_delete)>0:
#r.table('backlog').get_all(*tx_to_delete).delete(durability='soft').run(b.conn)
for transaction in tx_to_delete:
backlog.remove({'_id' : transaction})
if stop:
return
def bootstrap(self):
"""
Get transactions from the backlog that may have been assigned to this while it was
online (not listening to the changefeed)
"""
# create bigchain instance
b = Bigchain()
#inizializzo il client mongo
client=pymongo.MongoClient()
#indirizzo la collection backlog
backlog=client.bigchain.backlog
# create a queue to store initial results
q_initial = mp.Queue()
# get initial results
#initial_results = r.table('backlog').between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp').
#order_by(index=r.asc('assignee__transaction_timestamp')).run(b.conn)
'''
Nel momento in cui avviene il bootstrap e quindi l'accensione del bigchaindb viene analizzata la backlog per
rilevare tutte le transazioni non processate che sono state inserite in coda in attesa di essere processate
mentre il bigchaindb non era online.
Il porting da rethinkdb può avvenire con un semplice cursore che recupera tutti i documenti da backlog
ordinati secondo l'indice stabilito in 'db/utils.py'
'''
initial_results=[]
for tx in backlog.find( {
"$and":[
{ "assignee": { "$lte": b.me } } ,
{ "assignee": { "$gt": b.me } }
] } ).sort("assignee",pymongo.ASCENDING):
initial_results.append(tx)
# add results to the queue
for result in initial_results:
q_initial.put(result)
q_initial.put('stop')
return q_initial
def start(self):
"""
Bootstrap and start the processes
"""
logger.info('bootstraping block module...')
self.q_new_transaction = self.bootstrap()
logger.info('finished reading past transactions')
self._start()
logger.info('finished bootstraping block module...')
logger.info('starting block module...')
self.q_new_transaction = self._q_new_transaction
# signal initialization complete
self.initialized.set()
self._start()
logger.info('exiting block module...')
def _start(self):
"""
Initialize, spawn, and start the processes
"""
# initialize the processes
p_filter = mp.Process(name='filter_transactions', target=self.filter_by_assignee)
p_validate = mp.Process(name='validate_transactions', target=self.validate_transactions)
p_blocks = mp.Process(name='create_blocks', target=self.create_blocks)
p_write = mp.Process(name='write_blocks', target=self.write_blocks)
p_delete = mp.Process(name='delete_transactions', target=self.delete_transactions)
# start the processes
p_filter.start()
p_validate.start()
p_blocks.start()
p_write.start()
p_delete.start()
# join processes
p_filter.join()
p_validate.join()
p_blocks.join()
p_write.join()
p_delete.join()
class Block(object):
def __init__(self, q_new_transaction):
"""
Initialize the class with the needed
"""
self._q_new_transaction = q_new_transaction
self.q_new_transaction = None
self.q_tx_to_validate = mp.Queue()
self.q_tx_validated = mp.Queue()
self.q_tx_delete = mp.Queue()
self.q_block = mp.Queue()
self.initialized = mp.Event()
self.monitor = Monitor()
def filter_by_assignee(self):
"""
Handle transactions that are assigned to me
"""
# create a bigchain instance
b = Bigchain()
while True:
tx = self.q_new_transaction.get()
# poison pill
if tx == 'stop':
self.q_tx_to_validate.put('stop')
return
if tx['assignee'] == b.me:
tx.pop('assignee')
self.q_tx_to_validate.put(tx)
def validate_transactions(self):
"""
Checks if the incoming transactions are valid
"""
# create a bigchain instance
b = Bigchain()
while True:
self.monitor.gauge('tx_queue_gauge',
self.q_tx_to_validate.qsize(),
rate=bigchaindb.config['statsd']['rate'])
tx = self.q_tx_to_validate.get()
# poison pill
if tx == 'stop':
self.q_tx_delete.put('stop')
self.q_tx_validated.put('stop')
return
self.q_tx_delete.put(tx['id'])
with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']):
is_valid_transaction = b.is_valid_transaction(tx)
if is_valid_transaction:
self.q_tx_validated.put(tx)
def create_blocks(self):
"""
Create a block with valid transactions
"""
# create a bigchain instance
b = Bigchain()
stop = False
while True:
# read up to 1000 transactions
validated_transactions = []
for i in range(1000):
try:
tx = self.q_tx_validated.get(timeout=5)
except queue.Empty:
break
# poison pill
if tx == 'stop':
stop = True
break
validated_transactions.append(tx)
# if there are no transactions skip block creation
if validated_transactions:
# create block
block = b.create_block(validated_transactions)
self.q_block.put(block)
if stop:
self.q_block.put('stop')
return
def write_blocks(self):
"""
Write blocks to the bigchain
"""
# create bigchain instance
b = Bigchain()
# Write blocks
while True:
block = self.q_block.get()
# poison pill
if block == 'stop':
return
with self.monitor.timer('write_block'):
b.write_block(block)
def delete_transactions(self):
"""
Delete transactions from the backlog
"""
# create bigchain instance
b = Bigchain()
stop = False
while True:
# try to delete in batch to reduce io
tx_to_delete = []
for i in range(1000):
try:
tx = self.q_tx_delete.get(timeout=5)
except queue.Empty:
break
# poison pill
if tx == 'stop':
stop = True
break
tx_to_delete.append(tx)
if tx_to_delete:
r.table('backlog').get_all(*tx_to_delete).delete(durability='soft').run(b.conn)
if stop:
return
def bootstrap(self):
"""
Get transactions from the backlog that may have been assigned to this while it was
online (not listening to the changefeed)
"""
# create bigchain instance
b = Bigchain()
# create a queue to store initial results
q_initial = mp.Queue()
# get initial results
initial_results = r.table('backlog')\
.between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\
.order_by(index=r.asc('assignee__transaction_timestamp'))\
.run(b.conn)
# add results to the queue
for result in initial_results:
q_initial.put(result)
for i in range(mp.cpu_count()):
q_initial.put('stop')
return q_initial
def start(self):
"""
Bootstrap and start the processes
"""
logger.info('bootstraping block module...')
self.q_new_transaction = self.bootstrap()
logger.info('finished reading past transactions')
self._start()
logger.info('finished bootstraping block module...')
logger.info('starting block module...')
self.q_new_transaction = self._q_new_transaction
# signal initialization complete
self.initialized.set()
self._start()
logger.info('exiting block module...')
def kill(self):
for i in range(mp.cpu_count()):
self.q_new_transaction.put('stop')
def _start(self):
"""
Initialize, spawn, and start the processes
"""
# initialize the processes
p_filter = ProcessGroup(name='filter_transactions', target=self.filter_by_assignee)
p_validate = ProcessGroup(name='validate_transactions', target=self.validate_transactions)
p_blocks = ProcessGroup(name='create_blocks', target=self.create_blocks)
p_write = ProcessGroup(name='write_blocks', target=self.write_blocks)
p_delete = ProcessGroup(name='delete_transactions', target=self.delete_transactions)
# start the processes
p_filter.start()
p_validate.start()
p_blocks.start()
p_write.start()
p_delete.start()
class Block(object):
def __init__(self, q_new_transaction):
"""
Initialize the class with the needed
"""
self._q_new_transaction = q_new_transaction
self.q_new_transaction = None
self.q_tx_to_validate = mp.Queue()
self.q_tx_validated = mp.Queue()
self.q_tx_delete = mp.Queue()
self.q_block = mp.Queue()
self.initialized = mp.Event()
self.monitor = Monitor()
def filter_by_assignee(self):
"""
Handle transactions that are assigned to me
"""
# create a bigchain instance
b = Bigchain()
while True:
tx = self.q_new_transaction.get()
# poison pill
if tx == 'stop':
self.q_tx_to_validate.put('stop')
return
if tx['assignee'] == b.me:
tx.pop('assignee')
self.q_tx_to_validate.put(tx)
def validate_transactions(self):
"""
Checks if the incoming transactions are valid
"""
# create a bigchain instance
b = Bigchain()
while True:
self.monitor.gauge('tx_queue_gauge',
self.q_tx_to_validate.qsize(),
rate=bigchaindb.config['statsd']['rate'])
tx = self.q_tx_to_validate.get()
# poison pill
if tx == 'stop':
self.q_tx_delete.put('stop')
self.q_tx_validated.put('stop')
return
self.q_tx_delete.put(tx['id'])
with self.monitor.timer('validate_transaction',
rate=bigchaindb.config['statsd']['rate']):
is_valid_transaction = b.is_valid_transaction(tx)
if is_valid_transaction:
self.q_tx_validated.put(tx)
def create_blocks(self):
"""
Create a block with valid transactions
"""
# create a bigchain instance
b = Bigchain()
stop = False
while True:
# read up to 1000 transactions
validated_transactions = []
for i in range(1000):
try:
tx = self.q_tx_validated.get(timeout=5)
except queue.Empty:
break
# poison pill
if tx == 'stop':
stop = True
break
validated_transactions.append(tx)
# if there are no transactions skip block creation
if validated_transactions:
# create block
block = b.create_block(validated_transactions)
self.q_block.put(block)
if stop:
self.q_block.put('stop')
return
def write_blocks(self):
"""
Write blocks to the bigchain
"""
# create bigchain instance
b = Bigchain()
# Write blocks
while True:
block = self.q_block.get()
# poison pill
if block == 'stop':
return
with self.monitor.timer('write_block'):
b.write_block(block)
def delete_transactions(self):
"""
Delete transactions from the backlog
"""
# create bigchain instance
b = Bigchain()
stop = False
while True:
# try to delete in batch to reduce io
tx_to_delete = []
for i in range(1000):
try:
tx = self.q_tx_delete.get(timeout=5)
except queue.Empty:
break
# poison pill
if tx == 'stop':
stop = True
break
tx_to_delete.append(tx)
if tx_to_delete:
r.table('backlog').get_all(*tx_to_delete).delete(
durability='soft').run(b.conn)
if stop:
return
def bootstrap(self):
"""
Get transactions from the backlog that may have been assigned to this while it was
online (not listening to the changefeed)
"""
# create bigchain instance
b = Bigchain()
# create a queue to store initial results
q_initial = mp.Queue()
# get initial results
initial_results = r.table('backlog')\
.between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\
.order_by(index=r.asc('assignee__transaction_timestamp'))\
.run(b.conn)
# add results to the queue
for result in initial_results:
q_initial.put(result)
q_initial.put('stop')
return q_initial
def start(self):
"""
Bootstrap and start the processes
"""
logger.info('bootstraping block module...')
self.q_new_transaction = self.bootstrap()
logger.info('finished reading past transactions')
self._start()
logger.info('finished bootstraping block module...')
logger.info('starting block module...')
self.q_new_transaction = self._q_new_transaction
# signal initialization complete
self.initialized.set()
self._start()
logger.info('exiting block module...')
def _start(self):
"""
Initialize, spawn, and start the processes
"""
# initialize the processes
p_filter = mp.Process(name='filter_transactions',
target=self.filter_by_assignee)
p_validate = mp.Process(name='validate_transactions',
target=self.validate_transactions)
p_blocks = mp.Process(name='create_blocks', target=self.create_blocks)
p_write = mp.Process(name='write_blocks', target=self.write_blocks)
p_delete = mp.Process(name='delete_transactions',
target=self.delete_transactions)
# start the processes
p_filter.start()
p_validate.start()
p_blocks.start()
p_write.start()
p_delete.start()
# join processes
p_filter.join()
p_validate.join()
p_blocks.join()
p_write.join()
p_delete.join()
class Voter(object):
def __init__(self, q_new_block):
"""
Initialize the class with the needed queues.
Initialize with a queue where new blocks added to the bigchain will be put
"""
self.monitor = Monitor()
self.q_new_block = q_new_block
self.q_blocks_to_validate = mp.Queue()
self.q_validated_block = mp.Queue()
self.q_voted_block = mp.Queue()
self.v_previous_block_id = mp.Value(ctypes.c_char_p)
self.initialized = mp.Event()
def feed_blocks(self):
"""
Prepare the queue with blocks to validate
"""
block_stream = BlockStream(self.q_new_block)
while True:
# poison pill
block = block_stream.get()
if block == 'stop':
self.q_blocks_to_validate.put('stop')
return
self.q_blocks_to_validate.put(block)
def validate(self):
"""
Checks if incoming blocks are valid or not
"""
# create a bigchain instance. All processes should create their own bigchcain instance so that they all
# have their own connection to the database
b = Bigchain()
logger.info('voter waiting for new blocks')
# signal initialization complete
self.initialized.set()
while True:
new_block = self.q_blocks_to_validate.get()
# poison pill
if new_block == 'stop':
self.q_validated_block.put('stop')
return
logger.info('new_block arrived to voter')
with self.monitor.timer('validate_block'):
# FIXME: the following check is done also in `is_valid_block`,
# but validity can be true even if the block has already
# a vote.
if b.has_previous_vote(new_block):
continue
validity = b.is_valid_block(new_block)
self.q_validated_block.put((new_block,
self.v_previous_block_id.value.decode(),
validity))
self.v_previous_block_id.value = new_block['id'].encode()
def vote(self):
"""
Votes on the block based on the decision of the validation
"""
# create a bigchain instance
b = Bigchain()
while True:
elem = self.q_validated_block.get()
# poison pill
if elem == 'stop':
self.q_voted_block.put('stop')
return
validated_block, previous_block_id, decision = elem
vote = b.vote(validated_block['id'], previous_block_id, decision)
self.q_voted_block.put((validated_block, vote))
def update_block(self):
"""
Appends the vote in the bigchain table
"""
# create a bigchain instance
b = Bigchain()
while True:
elem = self.q_voted_block.get()
# poison pill
if elem == 'stop':
logger.info('clean exit')
return
block, vote = elem
pretty_vote = 'valid' if vote['vote']['is_block_valid'] else 'invalid'
logger.info('voting %s for block %s', pretty_vote, block['id'])
b.write_vote(block, vote)
def bootstrap(self):
"""
Before starting handling the new blocks received by the changefeed we need to handle unvoted blocks
added to the bigchain while the process was down
We also need to set the previous_block_id.
"""
b = Bigchain()
last_voted = b.get_last_voted_block()
self.v_previous_block_id.value = last_voted['id'].encode()
def kill(self):
"""
Terminate processes
"""
self.q_new_block.put('stop')
def start(self):
"""
Initialize, spawn, and start the processes
"""
self.bootstrap()
# initialize the processes
p_feed_blocks = mp.Process(name='block_feeder', target=self.feed_blocks)
p_validate = mp.Process(name='block_validator', target=self.validate)
p_vote = mp.Process(name='block_voter', target=self.vote)
p_update = mp.Process(name='block_updater', target=self.update_block)
# start the processes
p_feed_blocks.start()
p_validate.start()
p_vote.start()
p_update.start()
