class Voter(object): def __init__(self, q_new_block): """ Initialize the class with the needed queues. Initialize with a queue where new blocks added to the bigchain will be put """ self.monitor = Monitor() self.q_new_block = q_new_block self.q_blocks_to_validate = mp.Queue() self.q_validated_block = mp.Queue() self.q_voted_block = mp.Queue() self.v_previous_block_id = mp.Value(ctypes.c_char_p) self.v_previous_block_number = mp.Value(ctypes.c_uint64) self.initialized = mp.Event() def feed_blocks(self): """ Prepare the queue with blocks to validate """ block_stream = BlockStream(self.q_new_block) while True: # poison pill block = block_stream.get() if block == 'stop': self.q_blocks_to_validate.put('stop') return self.q_blocks_to_validate.put(block) def validate(self): """ Checks if incoming blocks are valid or not """ # create a bigchain instance. All processes should create their own bigchcain instance so that they all # have their own connection to the database b = Bigchain() logger.info('voter waiting for new blocks') # signal initialization complete self.initialized.set() while True: new_block = self.q_blocks_to_validate.get() # poison pill if new_block == 'stop': self.q_validated_block.put('stop') return logger.info('new_block arrived to voter') block_number = self.v_previous_block_number.value + 1 with self.monitor.timer('validate_block'): validity = b.is_valid_block(new_block) self.q_validated_block.put( (new_block, self.v_previous_block_id.value.decode(), block_number, validity)) self.v_previous_block_id.value = new_block['id'].encode() self.v_previous_block_number.value = block_number def vote(self): """ Votes on the block based on the decision of the validation """ # create a bigchain instance b = Bigchain() while True: elem = self.q_validated_block.get() # poison pill if elem == 'stop': self.q_voted_block.put('stop') return validated_block, previous_block_id, block_number, decision = elem vote = b.vote(validated_block, previous_block_id, decision) self.q_voted_block.put((validated_block, vote, block_number)) def update_block(self): """ Appends the vote in the bigchain table """ # create a bigchain instance b = Bigchain() while True: elem = self.q_voted_block.get() # poison pill if elem == 'stop': logger.info('clean exit') return block, vote, block_number = elem logger.info('updating block %s with number %s and with vote %s', block['id'], block_number, vote) b.write_vote(block, vote, block_number) def bootstrap(self): """ Before starting handling the new blocks received by the changefeed we need to handle unvoted blocks added to the bigchain while the process was down We also need to set the previous_block_id and the previous block_number """ b = Bigchain() last_voted = b.get_last_voted_block() self.v_previous_block_number.value = last_voted['block_number'] self.v_previous_block_id.value = last_voted['id'].encode() def kill(self): """ Terminate processes """ self.q_new_block.put('stop') def start(self): """ Initialize, spawn, and start the processes """ self.bootstrap() # initialize the processes p_feed_blocks = mp.Process(name='block_feeder', target=self.feed_blocks) p_validate = mp.Process(name='block_validator', target=self.validate) p_vote = mp.Process(name='block_voter', target=self.vote) p_update = mp.Process(name='block_updater', target=self.update_block) # start the processes p_feed_blocks.start() p_validate.start() p_vote.start() p_update.start()
class Block(object): def __init__(self, q_new_transaction): self._q_new_transaction = q_new_transaction self.q_new_transaction = None self.q_tx_to_validate = mp.Queue() self.q_tx_validated = mp.Queue() self.q_tx_delete = mp.Queue() self.q_block = mp.Queue() self.initialized = mp.Event() self.monitor = Monitor() def filter_by_assignee(self): """ Handle transactions that are assigned to me """ # create a bigchain instance b = Bigchain() while True: tx = self.q_new_transaction.get() # poison pill if tx == 'stop': self.q_tx_to_validate.put('stop') return if tx['assignee'] == b.me: tx.pop('assignee') self.q_tx_to_validate.put(tx) def validate_transactions(self): """ Checks if the incoming transactions are valid """ # create a bigchain instance b = Bigchain() while True: self.monitor.gauge('tx_queue_gauge', self.q_tx_to_validate.qsize(), rate=bigchaindb.config['statsd']['rate']) tx = self.q_tx_to_validate.get() # poison pill if tx == 'stop': self.q_tx_delete.put('stop') self.q_tx_validated.put('stop') return self.q_tx_delete.put(tx['_id']) with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']): is_valid_transaction = b.is_valid_transaction(tx) if is_valid_transaction: self.q_tx_validated.put(tx) def create_blocks(self): """ Create a block with valid transactions """ # create a bigchain instance b = Bigchain() stop = False while True: # read up to 1000 transactions validated_transactions = [] for i in range(1000): try: tx = self.q_tx_validated.get(timeout=5) except queue.Empty: break # poison pill if tx == 'stop': stop = True break validated_transactions.append(tx) # if there are no transactions skip block creation if validated_transactions: # create block block = b.create_block(validated_transactions) self.q_block.put(block) if stop: self.q_block.put('stop') return def write_blocks(self): """ Write blocks to the bigchain """ # create bigchain instance b = Bigchain() # Write blocks while True: block = self.q_block.get() # poison pill if block == 'stop': return with self.monitor.timer('write_block'): b.write_block(block) def delete_transactions(self): """ Delete transactions from the backlog """ # create bigchain instance b = Bigchain() client=pymongo.MongoClient() backlog=client.bigchain.backlog stop = False while True: # try to delete in batch to reduce io tx_to_delete = [] for i in range(1000): try: tx = self.q_tx_delete.get(timeout=5) except queue.Empty: break # poison pill if tx == 'stop': stop = True break tx_to_delete.append(tx) if len(tx_to_delete)>0: #r.table('backlog').get_all(*tx_to_delete).delete(durability='soft').run(b.conn) for transaction in tx_to_delete: backlog.remove({'_id' : transaction}) if stop: return def bootstrap(self): """ Get transactions from the backlog that may have been assigned to this while it was online (not listening to the changefeed) """ # create bigchain instance b = Bigchain() #inizializzo il client mongo client=pymongo.MongoClient() #indirizzo la collection backlog backlog=client.bigchain.backlog # create a queue to store initial results q_initial = mp.Queue() # get initial results #initial_results = r.table('backlog').between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp'). #order_by(index=r.asc('assignee__transaction_timestamp')).run(b.conn) ''' Nel momento in cui avviene il bootstrap e quindi l'accensione del bigchaindb viene analizzata la backlog per rilevare tutte le transazioni non processate che sono state inserite in coda in attesa di essere processate mentre il bigchaindb non era online. Il porting da rethinkdb può avvenire con un semplice cursore che recupera tutti i documenti da backlog ordinati secondo l'indice stabilito in 'db/utils.py' ''' initial_results=[] for tx in backlog.find( { "$and":[ { "assignee": { "$lte": b.me } } , { "assignee": { "$gt": b.me } } ] } ).sort("assignee",pymongo.ASCENDING): initial_results.append(tx) # add results to the queue for result in initial_results: q_initial.put(result) q_initial.put('stop') return q_initial def start(self): """ Bootstrap and start the processes """ logger.info('bootstraping block module...') self.q_new_transaction = self.bootstrap() logger.info('finished reading past transactions') self._start() logger.info('finished bootstraping block module...') logger.info('starting block module...') self.q_new_transaction = self._q_new_transaction # signal initialization complete self.initialized.set() self._start() logger.info('exiting block module...') def _start(self): """ Initialize, spawn, and start the processes """ # initialize the processes p_filter = mp.Process(name='filter_transactions', target=self.filter_by_assignee) p_validate = mp.Process(name='validate_transactions', target=self.validate_transactions) p_blocks = mp.Process(name='create_blocks', target=self.create_blocks) p_write = mp.Process(name='write_blocks', target=self.write_blocks) p_delete = mp.Process(name='delete_transactions', target=self.delete_transactions) # start the processes p_filter.start() p_validate.start() p_blocks.start() p_write.start() p_delete.start() # join processes p_filter.join() p_validate.join() p_blocks.join() p_write.join() p_delete.join()
class Block(object): def __init__(self, q_new_transaction): """ Initialize the class with the needed """ self._q_new_transaction = q_new_transaction self.q_new_transaction = None self.q_tx_to_validate = mp.Queue() self.q_tx_validated = mp.Queue() self.q_tx_delete = mp.Queue() self.q_block = mp.Queue() self.initialized = mp.Event() self.monitor = Monitor() def filter_by_assignee(self): """ Handle transactions that are assigned to me """ # create a bigchain instance b = Bigchain() while True: tx = self.q_new_transaction.get() # poison pill if tx == 'stop': self.q_tx_to_validate.put('stop') return if tx['assignee'] == b.me: tx.pop('assignee') self.q_tx_to_validate.put(tx) def validate_transactions(self): """ Checks if the incoming transactions are valid """ # create a bigchain instance b = Bigchain() while True: self.monitor.gauge('tx_queue_gauge', self.q_tx_to_validate.qsize(), rate=bigchaindb.config['statsd']['rate']) tx = self.q_tx_to_validate.get() # poison pill if tx == 'stop': self.q_tx_delete.put('stop') self.q_tx_validated.put('stop') return self.q_tx_delete.put(tx['id']) with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']): is_valid_transaction = b.is_valid_transaction(tx) if is_valid_transaction: self.q_tx_validated.put(tx) def create_blocks(self): """ Create a block with valid transactions """ # create a bigchain instance b = Bigchain() stop = False while True: # read up to 1000 transactions validated_transactions = [] for i in range(1000): try: tx = self.q_tx_validated.get(timeout=5) except queue.Empty: break # poison pill if tx == 'stop': stop = True break validated_transactions.append(tx) # if there are no transactions skip block creation if validated_transactions: # create block block = b.create_block(validated_transactions) self.q_block.put(block) if stop: self.q_block.put('stop') return def write_blocks(self): """ Write blocks to the bigchain """ # create bigchain instance b = Bigchain() # Write blocks while True: block = self.q_block.get() # poison pill if block == 'stop': return with self.monitor.timer('write_block'): b.write_block(block) def delete_transactions(self): """ Delete transactions from the backlog """ # create bigchain instance b = Bigchain() stop = False while True: # try to delete in batch to reduce io tx_to_delete = [] for i in range(1000): try: tx = self.q_tx_delete.get(timeout=5) except queue.Empty: break # poison pill if tx == 'stop': stop = True break tx_to_delete.append(tx) if tx_to_delete: r.table('backlog').get_all(*tx_to_delete).delete( durability='soft').run(b.conn) if stop: return def bootstrap(self): """ Get transactions from the backlog that may have been assigned to this while it was online (not listening to the changefeed) """ # create bigchain instance b = Bigchain() # create a queue to store initial results q_initial = mp.Queue() # get initial results initial_results = r.table('backlog')\ .between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\ .order_by(index=r.asc('assignee__transaction_timestamp'))\ .run(b.conn) # add results to the queue for result in initial_results: q_initial.put(result) q_initial.put('stop') return q_initial def start(self): """ Bootstrap and start the processes """ logger.info('bootstraping block module...') self.q_new_transaction = self.bootstrap() logger.info('finished reading past transactions') self._start() logger.info('finished bootstraping block module...') logger.info('starting block module...') self.q_new_transaction = self._q_new_transaction # signal initialization complete self.initialized.set() self._start() logger.info('exiting block module...') def _start(self): """ Initialize, spawn, and start the processes """ # initialize the processes p_filter = mp.Process(name='filter_transactions', target=self.filter_by_assignee) p_validate = mp.Process(name='validate_transactions', target=self.validate_transactions) p_blocks = mp.Process(name='create_blocks', target=self.create_blocks) p_write = mp.Process(name='write_blocks', target=self.write_blocks) p_delete = mp.Process(name='delete_transactions', target=self.delete_transactions) # start the processes p_filter.start() p_validate.start() p_blocks.start() p_write.start() p_delete.start() # join processes p_filter.join() p_validate.join() p_blocks.join() p_write.join() p_delete.join()
class Block(object): def __init__(self, q_new_transaction): """ Initialize the class with the needed """ self._q_new_transaction = q_new_transaction self.q_new_transaction = None self.q_tx_to_validate = mp.Queue() self.q_tx_validated = mp.Queue() self.q_tx_delete = mp.Queue() self.q_block = mp.Queue() self.initialized = mp.Event() self.monitor = Monitor() def filter_by_assignee(self): """ Handle transactions that are assigned to me """ # create a bigchain instance b = Bigchain() while True: tx = self.q_new_transaction.get() # poison pill if tx == 'stop': self.q_tx_to_validate.put('stop') return if tx['assignee'] == b.me: tx.pop('assignee') self.q_tx_to_validate.put(tx) def validate_transactions(self): """ Checks if the incoming transactions are valid """ # create a bigchain instance b = Bigchain() while True: self.monitor.gauge('tx_queue_gauge', self.q_tx_to_validate.qsize(), rate=bigchaindb.config['statsd']['rate']) tx = self.q_tx_to_validate.get() # poison pill if tx == 'stop': self.q_tx_delete.put('stop') self.q_tx_validated.put('stop') return self.q_tx_delete.put(tx['id']) with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']): is_valid_transaction = b.is_valid_transaction(tx) if is_valid_transaction: self.q_tx_validated.put(tx) def create_blocks(self): """ Create a block with valid transactions """ # create a bigchain instance b = Bigchain() stop = False while True: # read up to 1000 transactions validated_transactions = [] for i in range(1000): try: tx = self.q_tx_validated.get(timeout=5) except queue.Empty: break # poison pill if tx == 'stop': stop = True break validated_transactions.append(tx) # if there are no transactions skip block creation if validated_transactions: # create block block = b.create_block(validated_transactions) self.q_block.put(block) if stop: self.q_block.put('stop') return def write_blocks(self): """ Write blocks to the bigchain """ # create bigchain instance b = Bigchain() # Write blocks while True: block = self.q_block.get() # poison pill if block == 'stop': return with self.monitor.timer('write_block'): b.write_block(block) def delete_transactions(self): """ Delete transactions from the backlog """ # create bigchain instance b = Bigchain() stop = False while True: # try to delete in batch to reduce io tx_to_delete = [] for i in range(1000): try: tx = self.q_tx_delete.get(timeout=5) except queue.Empty: break # poison pill if tx == 'stop': stop = True break tx_to_delete.append(tx) if tx_to_delete: r.table('backlog').get_all(*tx_to_delete).delete(durability='soft').run(b.conn) if stop: return def bootstrap(self): """ Get transactions from the backlog that may have been assigned to this while it was online (not listening to the changefeed) """ # create bigchain instance b = Bigchain() # create a queue to store initial results q_initial = mp.Queue() # get initial results initial_results = r.table('backlog')\ .between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\ .order_by(index=r.asc('assignee__transaction_timestamp'))\ .run(b.conn) # add results to the queue for result in initial_results: q_initial.put(result) for i in range(mp.cpu_count()): q_initial.put('stop') return q_initial def start(self): """ Bootstrap and start the processes """ logger.info('bootstraping block module...') self.q_new_transaction = self.bootstrap() logger.info('finished reading past transactions') self._start() logger.info('finished bootstraping block module...') logger.info('starting block module...') self.q_new_transaction = self._q_new_transaction # signal initialization complete self.initialized.set() self._start() logger.info('exiting block module...') def kill(self): for i in range(mp.cpu_count()): self.q_new_transaction.put('stop') def _start(self): """ Initialize, spawn, and start the processes """ # initialize the processes p_filter = ProcessGroup(name='filter_transactions', target=self.filter_by_assignee) p_validate = ProcessGroup(name='validate_transactions', target=self.validate_transactions) p_blocks = ProcessGroup(name='create_blocks', target=self.create_blocks) p_write = ProcessGroup(name='write_blocks', target=self.write_blocks) p_delete = ProcessGroup(name='delete_transactions', target=self.delete_transactions) # start the processes p_filter.start() p_validate.start() p_blocks.start() p_write.start() p_delete.start()
class Voter(object): def __init__(self, q_new_block): """ Initialize the class with the needed queues. Initialize with a queue where new blocks added to the bigchain will be put """ self.monitor = Monitor() self.q_new_block = q_new_block self.q_blocks_to_validate = mp.Queue() self.q_validated_block = mp.Queue() self.q_voted_block = mp.Queue() self.v_previous_block_id = mp.Value(ctypes.c_char_p) self.initialized = mp.Event() def feed_blocks(self): """ Prepare the queue with blocks to validate """ block_stream = BlockStream(self.q_new_block) while True: # poison pill block = block_stream.get() if block == 'stop': self.q_blocks_to_validate.put('stop') return self.q_blocks_to_validate.put(block) def validate(self): """ Checks if incoming blocks are valid or not """ # create a bigchain instance. All processes should create their own bigchcain instance so that they all # have their own connection to the database b = Bigchain() logger.info('voter waiting for new blocks') # signal initialization complete self.initialized.set() while True: new_block = self.q_blocks_to_validate.get() # poison pill if new_block == 'stop': self.q_validated_block.put('stop') return logger.info('new_block arrived to voter') with self.monitor.timer('validate_block'): # FIXME: the following check is done also in `is_valid_block`, # but validity can be true even if the block has already # a vote. if b.has_previous_vote(new_block): continue validity = b.is_valid_block(new_block) self.q_validated_block.put((new_block, self.v_previous_block_id.value.decode(), validity)) self.v_previous_block_id.value = new_block['id'].encode() def vote(self): """ Votes on the block based on the decision of the validation """ # create a bigchain instance b = Bigchain() while True: elem = self.q_validated_block.get() # poison pill if elem == 'stop': self.q_voted_block.put('stop') return validated_block, previous_block_id, decision = elem vote = b.vote(validated_block['id'], previous_block_id, decision) self.q_voted_block.put((validated_block, vote)) def update_block(self): """ Appends the vote in the bigchain table """ # create a bigchain instance b = Bigchain() while True: elem = self.q_voted_block.get() # poison pill if elem == 'stop': logger.info('clean exit') return block, vote = elem pretty_vote = 'valid' if vote['vote']['is_block_valid'] else 'invalid' logger.info('voting %s for block %s', pretty_vote, block['id']) b.write_vote(block, vote) def bootstrap(self): """ Before starting handling the new blocks received by the changefeed we need to handle unvoted blocks added to the bigchain while the process was down We also need to set the previous_block_id. """ b = Bigchain() last_voted = b.get_last_voted_block() self.v_previous_block_id.value = last_voted['id'].encode() def kill(self): """ Terminate processes """ self.q_new_block.put('stop') def start(self): """ Initialize, spawn, and start the processes """ self.bootstrap() # initialize the processes p_feed_blocks = mp.Process(name='block_feeder', target=self.feed_blocks) p_validate = mp.Process(name='block_validator', target=self.validate) p_vote = mp.Process(name='block_voter', target=self.vote) p_update = mp.Process(name='block_updater', target=self.update_block) # start the processes p_feed_blocks.start() p_validate.start() p_vote.start() p_update.start()