Ejemplo n.º 1
0
class Voter(object):
    def __init__(self, q_new_block):
        """
        Initialize the class with the needed queues.

        Initialize with a queue where new blocks added to the bigchain will be put
        """

        self.monitor = Monitor()

        self.q_new_block = q_new_block
        self.q_blocks_to_validate = mp.Queue()
        self.q_validated_block = mp.Queue()
        self.q_voted_block = mp.Queue()
        self.v_previous_block_id = mp.Value(ctypes.c_char_p)
        self.v_previous_block_number = mp.Value(ctypes.c_uint64)
        self.initialized = mp.Event()

    def feed_blocks(self):
        """
        Prepare the queue with blocks to validate
        """

        block_stream = BlockStream(self.q_new_block)
        while True:
            # poison pill
            block = block_stream.get()
            if block == 'stop':
                self.q_blocks_to_validate.put('stop')
                return

            self.q_blocks_to_validate.put(block)

    def validate(self):
        """
        Checks if incoming blocks are valid or not
        """

        # create a bigchain instance. All processes should create their own bigchcain instance so that they all
        # have their own connection to the database
        b = Bigchain()

        logger.info('voter waiting for new blocks')
        # signal initialization complete
        self.initialized.set()

        while True:
            new_block = self.q_blocks_to_validate.get()

            # poison pill
            if new_block == 'stop':
                self.q_validated_block.put('stop')
                return

            logger.info('new_block arrived to voter')
            block_number = self.v_previous_block_number.value + 1

            with self.monitor.timer('validate_block'):
                validity = b.is_valid_block(new_block)

            self.q_validated_block.put(
                (new_block, self.v_previous_block_id.value.decode(),
                 block_number, validity))

            self.v_previous_block_id.value = new_block['id'].encode()
            self.v_previous_block_number.value = block_number

    def vote(self):
        """
        Votes on the block based on the decision of the validation
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            elem = self.q_validated_block.get()

            # poison pill
            if elem == 'stop':
                self.q_voted_block.put('stop')
                return

            validated_block, previous_block_id, block_number, decision = elem
            vote = b.vote(validated_block, previous_block_id, decision)
            self.q_voted_block.put((validated_block, vote, block_number))

    def update_block(self):
        """
        Appends the vote in the bigchain table
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            elem = self.q_voted_block.get()

            # poison pill
            if elem == 'stop':
                logger.info('clean exit')
                return

            block, vote, block_number = elem
            logger.info('updating block %s with number %s and with vote %s',
                        block['id'], block_number, vote)
            b.write_vote(block, vote, block_number)

    def bootstrap(self):
        """
        Before starting handling the new blocks received by the changefeed we need to handle unvoted blocks
        added to the bigchain while the process was down

        We also need to set the previous_block_id and the previous block_number
        """

        b = Bigchain()
        last_voted = b.get_last_voted_block()

        self.v_previous_block_number.value = last_voted['block_number']
        self.v_previous_block_id.value = last_voted['id'].encode()

    def kill(self):
        """
        Terminate processes
        """
        self.q_new_block.put('stop')

    def start(self):
        """
        Initialize, spawn, and start the processes
        """

        self.bootstrap()

        # initialize the processes
        p_feed_blocks = mp.Process(name='block_feeder',
                                   target=self.feed_blocks)
        p_validate = mp.Process(name='block_validator', target=self.validate)
        p_vote = mp.Process(name='block_voter', target=self.vote)
        p_update = mp.Process(name='block_updater', target=self.update_block)

        # start the processes
        p_feed_blocks.start()
        p_validate.start()
        p_vote.start()
        p_update.start()
Ejemplo n.º 2
0
class Block(object):
    def __init__(self, q_new_transaction):
        self._q_new_transaction = q_new_transaction
        self.q_new_transaction = None
        self.q_tx_to_validate = mp.Queue()
        self.q_tx_validated = mp.Queue()
        self.q_tx_delete = mp.Queue()
        self.q_block = mp.Queue()
        self.initialized = mp.Event()
        self.monitor = Monitor()

    def filter_by_assignee(self):
        """
        Handle transactions that are assigned to me
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            tx = self.q_new_transaction.get()

            # poison pill
            if tx == 'stop':
                self.q_tx_to_validate.put('stop')
                return

            if tx['assignee'] == b.me:
                tx.pop('assignee')
                self.q_tx_to_validate.put(tx)

    def validate_transactions(self):
        """
        Checks if the incoming transactions are valid
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            self.monitor.gauge('tx_queue_gauge',
                               self.q_tx_to_validate.qsize(),
                               rate=bigchaindb.config['statsd']['rate'])
            tx = self.q_tx_to_validate.get()

            # poison pill
            if tx == 'stop':
                self.q_tx_delete.put('stop')
                self.q_tx_validated.put('stop')
                return

            self.q_tx_delete.put(tx['_id'])

            with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']):
                is_valid_transaction = b.is_valid_transaction(tx)

            if is_valid_transaction:
                self.q_tx_validated.put(tx)

    def create_blocks(self):
        """
        Create a block with valid transactions
        """

        # create a bigchain instance
        b = Bigchain()
        stop = False

        while True:

            # read up to 1000 transactions
            validated_transactions = []
            for i in range(1000):
                try:
                    tx = self.q_tx_validated.get(timeout=5)
                except queue.Empty:
                    break

                # poison pill
                if tx == 'stop':
                    stop = True
                    break

                validated_transactions.append(tx)

            # if there are no transactions skip block creation
            if validated_transactions:
                # create block
                block = b.create_block(validated_transactions)
                self.q_block.put(block)

            if stop:
                self.q_block.put('stop')
                return

    def write_blocks(self):
        """
        Write blocks to the bigchain
        """

        # create bigchain instance
        b = Bigchain()

        # Write blocks
        while True:
            block = self.q_block.get()

            # poison pill
            if block == 'stop':
                return

            with self.monitor.timer('write_block'):
                b.write_block(block)

    def delete_transactions(self):
        """
        Delete transactions from the backlog
        """
        # create bigchain instance
        b = Bigchain()
        client=pymongo.MongoClient()
        backlog=client.bigchain.backlog
        stop = False

        while True:
            # try to delete in batch to reduce io
            tx_to_delete = []
            for i in range(1000):
                try:
                    tx = self.q_tx_delete.get(timeout=5)
                except queue.Empty:
                    break

                # poison pill
                if tx == 'stop':
                    stop = True
                    break
                tx_to_delete.append(tx)

            if len(tx_to_delete)>0:
                #r.table('backlog').get_all(*tx_to_delete).delete(durability='soft').run(b.conn)
                for transaction in tx_to_delete:
                    backlog.remove({'_id' : transaction})
            if stop:
                return


    def bootstrap(self):
        """
        Get transactions from the backlog that may have been assigned to this while it was
        online (not listening to the changefeed)
        """
        # create bigchain instance
        b = Bigchain()
        #inizializzo il client mongo
        client=pymongo.MongoClient()
        #indirizzo la collection backlog
        backlog=client.bigchain.backlog

        # create a queue to store initial results
        q_initial = mp.Queue()

        # get initial results
        #initial_results = r.table('backlog').between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp').
                            #order_by(index=r.asc('assignee__transaction_timestamp')).run(b.conn)
        '''
        Nel momento in cui avviene il bootstrap e quindi l'accensione del bigchaindb viene analizzata la backlog per 
        rilevare tutte le transazioni non processate che sono state inserite in coda in attesa di essere processate 
        mentre il bigchaindb non era online.

        Il porting da rethinkdb può avvenire con un semplice cursore che recupera tutti i documenti da backlog
        ordinati secondo l'indice stabilito in 'db/utils.py'
        '''

        initial_results=[]
        for tx in backlog.find( { 
            "$and":[ 
                { "assignee": { "$lte": b.me } } , 
                { "assignee": { "$gt": b.me } } 
                ] } ).sort("assignee",pymongo.ASCENDING):
        
            initial_results.append(tx)


        # add results to the queue
        for result in initial_results:
            q_initial.put(result)
        q_initial.put('stop')

        return q_initial



    def start(self):
        """
        Bootstrap and start the processes
        """
        logger.info('bootstraping block module...')
        self.q_new_transaction = self.bootstrap()
        logger.info('finished reading past transactions')
        self._start()
        logger.info('finished bootstraping block module...')

        logger.info('starting block module...')
        self.q_new_transaction = self._q_new_transaction

        # signal initialization complete
        self.initialized.set()

        self._start()
        logger.info('exiting block module...')

    def _start(self):
        """
        Initialize, spawn, and start the processes
        """

        # initialize the processes
        p_filter = mp.Process(name='filter_transactions', target=self.filter_by_assignee)
        p_validate = mp.Process(name='validate_transactions', target=self.validate_transactions)
        p_blocks = mp.Process(name='create_blocks', target=self.create_blocks)
        p_write = mp.Process(name='write_blocks', target=self.write_blocks)
        p_delete = mp.Process(name='delete_transactions', target=self.delete_transactions)

        # start the processes
        p_filter.start()
        p_validate.start()
        p_blocks.start()
        p_write.start()
        p_delete.start()

        # join processes
        p_filter.join()
        p_validate.join()
        p_blocks.join()
        p_write.join()
        p_delete.join()
Ejemplo n.º 3
0
class Block(object):
    def __init__(self, q_new_transaction):
        """
        Initialize the class with the needed
        """
        self._q_new_transaction = q_new_transaction
        self.q_new_transaction = None
        self.q_tx_to_validate = mp.Queue()
        self.q_tx_validated = mp.Queue()
        self.q_tx_delete = mp.Queue()
        self.q_block = mp.Queue()
        self.initialized = mp.Event()
        self.monitor = Monitor()

    def filter_by_assignee(self):
        """
        Handle transactions that are assigned to me
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            tx = self.q_new_transaction.get()

            # poison pill
            if tx == 'stop':
                self.q_tx_to_validate.put('stop')
                return

            if tx['assignee'] == b.me:
                tx.pop('assignee')
                self.q_tx_to_validate.put(tx)

    def validate_transactions(self):
        """
        Checks if the incoming transactions are valid
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            self.monitor.gauge('tx_queue_gauge',
                               self.q_tx_to_validate.qsize(),
                               rate=bigchaindb.config['statsd']['rate'])
            tx = self.q_tx_to_validate.get()

            # poison pill
            if tx == 'stop':
                self.q_tx_delete.put('stop')
                self.q_tx_validated.put('stop')
                return

            self.q_tx_delete.put(tx['id'])

            with self.monitor.timer('validate_transaction',
                                    rate=bigchaindb.config['statsd']['rate']):
                is_valid_transaction = b.is_valid_transaction(tx)

            if is_valid_transaction:
                self.q_tx_validated.put(tx)

    def create_blocks(self):
        """
        Create a block with valid transactions
        """

        # create a bigchain instance
        b = Bigchain()
        stop = False

        while True:

            # read up to 1000 transactions
            validated_transactions = []
            for i in range(1000):
                try:
                    tx = self.q_tx_validated.get(timeout=5)
                except queue.Empty:
                    break

                # poison pill
                if tx == 'stop':
                    stop = True
                    break

                validated_transactions.append(tx)

            # if there are no transactions skip block creation
            if validated_transactions:
                # create block
                block = b.create_block(validated_transactions)
                self.q_block.put(block)

            if stop:
                self.q_block.put('stop')
                return

    def write_blocks(self):
        """
        Write blocks to the bigchain
        """

        # create bigchain instance
        b = Bigchain()

        # Write blocks
        while True:
            block = self.q_block.get()

            # poison pill
            if block == 'stop':
                return

            with self.monitor.timer('write_block'):
                b.write_block(block)

    def delete_transactions(self):
        """
        Delete transactions from the backlog
        """
        # create bigchain instance
        b = Bigchain()
        stop = False

        while True:
            # try to delete in batch to reduce io
            tx_to_delete = []
            for i in range(1000):
                try:
                    tx = self.q_tx_delete.get(timeout=5)
                except queue.Empty:
                    break

                # poison pill
                if tx == 'stop':
                    stop = True
                    break

                tx_to_delete.append(tx)

            if tx_to_delete:
                r.table('backlog').get_all(*tx_to_delete).delete(
                    durability='soft').run(b.conn)

            if stop:
                return

    def bootstrap(self):
        """
        Get transactions from the backlog that may have been assigned to this while it was
        online (not listening to the changefeed)
        """
        # create bigchain instance
        b = Bigchain()

        # create a queue to store initial results
        q_initial = mp.Queue()

        # get initial results
        initial_results = r.table('backlog')\
            .between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\
            .order_by(index=r.asc('assignee__transaction_timestamp'))\
            .run(b.conn)

        # add results to the queue
        for result in initial_results:
            q_initial.put(result)
        q_initial.put('stop')

        return q_initial

    def start(self):
        """
        Bootstrap and start the processes
        """
        logger.info('bootstraping block module...')
        self.q_new_transaction = self.bootstrap()
        logger.info('finished reading past transactions')
        self._start()
        logger.info('finished bootstraping block module...')

        logger.info('starting block module...')
        self.q_new_transaction = self._q_new_transaction

        # signal initialization complete
        self.initialized.set()

        self._start()
        logger.info('exiting block module...')

    def _start(self):
        """
        Initialize, spawn, and start the processes
        """

        # initialize the processes
        p_filter = mp.Process(name='filter_transactions',
                              target=self.filter_by_assignee)
        p_validate = mp.Process(name='validate_transactions',
                                target=self.validate_transactions)
        p_blocks = mp.Process(name='create_blocks', target=self.create_blocks)
        p_write = mp.Process(name='write_blocks', target=self.write_blocks)
        p_delete = mp.Process(name='delete_transactions',
                              target=self.delete_transactions)

        # start the processes
        p_filter.start()
        p_validate.start()
        p_blocks.start()
        p_write.start()
        p_delete.start()

        # join processes
        p_filter.join()
        p_validate.join()
        p_blocks.join()
        p_write.join()
        p_delete.join()
Ejemplo n.º 4
0
class Block(object):

    def __init__(self, q_new_transaction):
        """
        Initialize the class with the needed
        """
        self._q_new_transaction = q_new_transaction
        self.q_new_transaction = None
        self.q_tx_to_validate = mp.Queue()
        self.q_tx_validated = mp.Queue()
        self.q_tx_delete = mp.Queue()
        self.q_block = mp.Queue()
        self.initialized = mp.Event()
        self.monitor = Monitor()

    def filter_by_assignee(self):
        """
        Handle transactions that are assigned to me
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            tx = self.q_new_transaction.get()

            # poison pill
            if tx == 'stop':
                self.q_tx_to_validate.put('stop')
                return

            if tx['assignee'] == b.me:
                tx.pop('assignee')
                self.q_tx_to_validate.put(tx)

    def validate_transactions(self):
        """
        Checks if the incoming transactions are valid
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            self.monitor.gauge('tx_queue_gauge',
                               self.q_tx_to_validate.qsize(),
                               rate=bigchaindb.config['statsd']['rate'])
            tx = self.q_tx_to_validate.get()

            # poison pill
            if tx == 'stop':
                self.q_tx_delete.put('stop')
                self.q_tx_validated.put('stop')
                return

            self.q_tx_delete.put(tx['id'])

            with self.monitor.timer('validate_transaction', rate=bigchaindb.config['statsd']['rate']):
                is_valid_transaction = b.is_valid_transaction(tx)

            if is_valid_transaction:
                self.q_tx_validated.put(tx)

    def create_blocks(self):
        """
        Create a block with valid transactions
        """

        # create a bigchain instance
        b = Bigchain()
        stop = False

        while True:

            # read up to 1000 transactions
            validated_transactions = []
            for i in range(1000):
                try:
                    tx = self.q_tx_validated.get(timeout=5)
                except queue.Empty:
                    break

                # poison pill
                if tx == 'stop':
                    stop = True
                    break

                validated_transactions.append(tx)

            # if there are no transactions skip block creation
            if validated_transactions:
                # create block
                block = b.create_block(validated_transactions)
                self.q_block.put(block)

            if stop:
                self.q_block.put('stop')
                return

    def write_blocks(self):
        """
        Write blocks to the bigchain
        """

        # create bigchain instance
        b = Bigchain()

        # Write blocks
        while True:
            block = self.q_block.get()

            # poison pill
            if block == 'stop':
                return

            with self.monitor.timer('write_block'):
                b.write_block(block)

    def delete_transactions(self):
        """
        Delete transactions from the backlog
        """
        # create bigchain instance
        b = Bigchain()
        stop = False

        while True:
            # try to delete in batch to reduce io
            tx_to_delete = []
            for i in range(1000):
                try:
                    tx = self.q_tx_delete.get(timeout=5)
                except queue.Empty:
                    break

                # poison pill
                if tx == 'stop':
                    stop = True
                    break

                tx_to_delete.append(tx)

            if tx_to_delete:
                r.table('backlog').get_all(*tx_to_delete).delete(durability='soft').run(b.conn)

            if stop:
                return

    def bootstrap(self):
        """
        Get transactions from the backlog that may have been assigned to this while it was
        online (not listening to the changefeed)
        """
        # create bigchain instance
        b = Bigchain()

        # create a queue to store initial results
        q_initial = mp.Queue()

        # get initial results
        initial_results = r.table('backlog')\
            .between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\
            .order_by(index=r.asc('assignee__transaction_timestamp'))\
            .run(b.conn)

        # add results to the queue
        for result in initial_results:
            q_initial.put(result)

        for i in range(mp.cpu_count()):
            q_initial.put('stop')

        return q_initial

    def start(self):
        """
        Bootstrap and start the processes
        """
        logger.info('bootstraping block module...')
        self.q_new_transaction = self.bootstrap()
        logger.info('finished reading past transactions')
        self._start()
        logger.info('finished bootstraping block module...')

        logger.info('starting block module...')
        self.q_new_transaction = self._q_new_transaction

        # signal initialization complete
        self.initialized.set()

        self._start()
        logger.info('exiting block module...')

    def kill(self):
        for i in range(mp.cpu_count()):
            self.q_new_transaction.put('stop')

    def _start(self):
        """
        Initialize, spawn, and start the processes
        """

        # initialize the processes
        p_filter = ProcessGroup(name='filter_transactions', target=self.filter_by_assignee)
        p_validate = ProcessGroup(name='validate_transactions', target=self.validate_transactions)
        p_blocks = ProcessGroup(name='create_blocks', target=self.create_blocks)
        p_write = ProcessGroup(name='write_blocks', target=self.write_blocks)
        p_delete = ProcessGroup(name='delete_transactions', target=self.delete_transactions)

        # start the processes
        p_filter.start()
        p_validate.start()
        p_blocks.start()
        p_write.start()
        p_delete.start()
Ejemplo n.º 5
0
class Voter(object):

    def __init__(self, q_new_block):
        """
        Initialize the class with the needed queues.

        Initialize with a queue where new blocks added to the bigchain will be put
        """

        self.monitor = Monitor()

        self.q_new_block = q_new_block
        self.q_blocks_to_validate = mp.Queue()
        self.q_validated_block = mp.Queue()
        self.q_voted_block = mp.Queue()
        self.v_previous_block_id = mp.Value(ctypes.c_char_p)
        self.initialized = mp.Event()

    def feed_blocks(self):
        """
        Prepare the queue with blocks to validate
        """

        block_stream = BlockStream(self.q_new_block)
        while True:
            # poison pill
            block = block_stream.get()
            if block == 'stop':
                self.q_blocks_to_validate.put('stop')
                return

            self.q_blocks_to_validate.put(block)

    def validate(self):
        """
        Checks if incoming blocks are valid or not
        """

        # create a bigchain instance. All processes should create their own bigchcain instance so that they all
        # have their own connection to the database
        b = Bigchain()

        logger.info('voter waiting for new blocks')
        # signal initialization complete
        self.initialized.set()

        while True:
            new_block = self.q_blocks_to_validate.get()

            # poison pill
            if new_block == 'stop':
                self.q_validated_block.put('stop')
                return

            logger.info('new_block arrived to voter')

            with self.monitor.timer('validate_block'):
                # FIXME: the following check is done also in `is_valid_block`,
                #        but validity can be true even if the block has already
                #        a vote.
                if b.has_previous_vote(new_block):
                    continue
                validity = b.is_valid_block(new_block)

            self.q_validated_block.put((new_block,
                                        self.v_previous_block_id.value.decode(),
                                        validity))

            self.v_previous_block_id.value = new_block['id'].encode()

    def vote(self):
        """
        Votes on the block based on the decision of the validation
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            elem = self.q_validated_block.get()

            # poison pill
            if elem == 'stop':
                self.q_voted_block.put('stop')
                return

            validated_block, previous_block_id, decision = elem
            vote = b.vote(validated_block['id'], previous_block_id, decision)
            self.q_voted_block.put((validated_block, vote))

    def update_block(self):
        """
        Appends the vote in the bigchain table
        """

        # create a bigchain instance
        b = Bigchain()

        while True:
            elem = self.q_voted_block.get()

            # poison pill
            if elem == 'stop':
                logger.info('clean exit')
                return

            block, vote = elem
            pretty_vote = 'valid' if vote['vote']['is_block_valid'] else 'invalid'
            logger.info('voting %s for block %s', pretty_vote, block['id'])
            b.write_vote(block, vote)

    def bootstrap(self):
        """
        Before starting handling the new blocks received by the changefeed we need to handle unvoted blocks
        added to the bigchain while the process was down

        We also need to set the previous_block_id.
        """

        b = Bigchain()
        last_voted = b.get_last_voted_block()

        self.v_previous_block_id.value = last_voted['id'].encode()

    def kill(self):
        """
        Terminate processes
        """
        self.q_new_block.put('stop')

    def start(self):
        """
        Initialize, spawn, and start the processes
        """

        self.bootstrap()

        # initialize the processes
        p_feed_blocks = mp.Process(name='block_feeder', target=self.feed_blocks)
        p_validate = mp.Process(name='block_validator', target=self.validate)
        p_vote = mp.Process(name='block_voter', target=self.vote)
        p_update = mp.Process(name='block_updater', target=self.update_block)

        # start the processes
        p_feed_blocks.start()
        p_validate.start()
        p_vote.start()
        p_update.start()