Example #1
0
 def run_client(peer_list, tx_list):
     tx_store = {}
     inv_collector = InvCollector()
     tx_handler = TxHandler(inv_collector, tx_store)
     for tx in tx_list:
         tx_handler.add_tx(tx)
     for peer in peer_list:
         inv_collector.add_peer(peer)
         tx_handler.add_peer(peer)
     while len(tx_store) < 20:
         yield from asyncio.sleep(0.1)
     return tx_store
Example #2
0
 def run_client(peer_list, tx_list):
     tx_store = {}
     inv_collector = InvCollector()
     tx_handler = TxHandler(inv_collector, tx_store)
     for tx in tx_list:
         tx_handler.add_tx(tx)
     for peer in peer_list:
         inv_collector.add_peer(peer)
         tx_handler.add_peer(peer)
     while len(tx_store) < 20:
         yield from asyncio.sleep(0.1)
     return tx_store
Example #3
0
def test_InvCollector():
    # create some peers
    peer1_2, peer2 = create_handshaked_peers()
    peer1_3, peer3 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.3")
    peer1_4, peer4 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.4")

    # peer1_* represents the local peer

    TX_LIST = [make_tx(i) for i in range(100)]

    @asyncio.coroutine
    def run_local_peer(inv_collector, inv_item_q):
        r = []
        while len(r) < 90:
            inv_item = yield from inv_item_q.get()
            v = yield from inv_collector.fetch(inv_item)
            r.append(v)
        return r

    @asyncio.coroutine
    def run_remote_peer(next_message, peer, txs):
        tx_db = dict((tx.hash(), tx) for tx in txs)
        r = []

        inv_items = [InvItem(ITEM_TYPE_TX, tx.hash()) for tx in txs]
        peer.send_msg("inv", items=inv_items)

        while True:
            t = yield from next_message()
            r.append(t)
            if t[0] == 'getdata':
                for inv_item in t[1]["items"]:
                    peer.send_msg("tx", tx=tx_db[inv_item.data])
        return r

    futures = []
    for peer, txs in [(peer2, TX_LIST[:30]), (peer3, TX_LIST[30:60]),
                      (peer4, TX_LIST[60:90])]:
        f = asyncio.Task(
            run_remote_peer(peer.new_get_next_message_f(), peer, txs))
        futures.append(f)

    inv_collector = InvCollector()
    [inv_collector.add_peer(peer) for peer in [peer1_2, peer1_3, peer1_4]]

    f = asyncio.Task(
        run_local_peer(inv_collector, inv_collector.new_inv_item_queue()))
    done, pending = asyncio.get_event_loop().run_until_complete(
        asyncio.wait([f], timeout=5.0))
    r = done.pop().result()
    assert len(r) == 90
    assert [tx.hash() for tx in r] == [tx.hash() for tx in TX_LIST[:90]]
 def run_client(peer_list, block_list):
     block_chain = BlockChain()
     block_store = {}
     inv_collector = InvCollector()
     block_handler = BlockHandler(inv_collector, block_chain, block_store)
     for peer in peer_list:
         inv_collector.add_peer(peer)
         block_handler.add_peer(peer)
     for block in block_list:
         inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
         block_store[block.hash()] = block
     while len(block_store) < 2:
         yield from asyncio.sleep(0.1)
     return block_store
Example #5
0
def test_InvCollector():
    # create some peers
    peer1_2, peer2 = create_handshaked_peers()
    peer1_3, peer3 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.3")
    peer1_4, peer4 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.4")

    # peer1_* represents the local peer

    TX_LIST = [make_tx(i) for i in range(100)]

    @asyncio.coroutine
    def run_local_peer(inv_collector, inv_item_q):
        r = []
        while len(r) < 90:
            inv_item = yield from inv_item_q.get()
            v = yield from inv_collector.fetch(inv_item)
            r.append(v)
        return r

    @asyncio.coroutine
    def run_remote_peer(next_message, peer, txs):
        tx_db = dict((tx.hash(), tx) for tx in txs)
        r = []

        inv_items = [InvItem(ITEM_TYPE_TX, tx.hash()) for tx in txs]
        peer.send_msg("inv", items=inv_items)

        while True:
            t = yield from next_message()
            r.append(t)
            if t[0] == 'getdata':
                for inv_item in t[1]["items"]:
                    peer.send_msg("tx", tx=tx_db[inv_item.data])
        return r

    futures = []
    for peer, txs in [(peer2, TX_LIST[:30]), (peer3, TX_LIST[30:60]), (peer4, TX_LIST[60:90])]:
        f = asyncio.Task(run_remote_peer(peer.new_get_next_message_f(), peer, txs))
        futures.append(f)

    inv_collector = InvCollector()
    [inv_collector.add_peer(peer) for peer in [peer1_2, peer1_3, peer1_4]]

    f = asyncio.Task(run_local_peer(inv_collector, inv_collector.new_inv_item_queue()))
    done, pending = asyncio.get_event_loop().run_until_complete(asyncio.wait([f], timeout=5.0))
    r = done.pop().result()
    assert len(r) == 90
    assert [tx.hash() for tx in r] == [tx.hash() for tx in TX_LIST[:90]]
def items_for_client(initial_blocks=[]):
    block_store = {}
    block_chain = BlockChain()
    blockfetcher = Blockfetcher()
    inv_collector = InvCollector()
    block_handler = BlockHandler(inv_collector, block_chain, block_store)
    fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)

    for block in initial_blocks:
        inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
        block_store[block.hash()] = block
    block_chain.add_headers(initial_blocks)

    inv_q = inv_collector.new_inv_item_queue()
    ap = make_add_peer(fast_forward_add_peer, blockfetcher, block_handler, inv_collector, block_chain, block_store)
    ap.block_getter_task = asyncio.Task(block_getter(inv_q, inv_collector, block_handler, block_chain, block_store))
    return block_handler, block_chain, block_store, ap
Example #7
0
    def run_local_peer(peer_list, r):
        inv_collector = InvCollector()
        for peer in peer_list:
            inv_collector.add_peer(peer)
        inv_item_q = inv_collector.new_inv_item_queue()
        
        @asyncio.coroutine
        def _do_fetch(inv_collector, inv_item, r):
            v = yield from inv_collector.fetch(inv_item, peer_timeout=3.0)
            if v:
                r.append(v)

        # keep a strong reference to these tasks
        tasks = []
        for i in range(10):
            inv_item = yield from inv_item_q.get()
            tasks.append(asyncio.Task(_do_fetch(inv_collector, inv_item, r)))
        while len(r) < 10:
            yield from asyncio.sleep(0.1)
        return r
Example #8
0
def items_for_client(initial_blocks=[]):
    block_store = {}
    block_chain = BlockChain()
    blockfetcher = Blockfetcher()
    inv_collector = InvCollector()
    block_handler = BlockHandler(inv_collector, block_chain, block_store)
    fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)

    for block in initial_blocks:
        inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
        block_store[block.hash()] = block
    block_chain.add_headers(initial_blocks)

    inv_q = inv_collector.new_inv_item_queue()
    ap = make_add_peer(fast_forward_add_peer, blockfetcher, block_handler,
                       inv_collector, block_chain, block_store)
    ap.block_getter_task = asyncio.Task(
        block_getter(inv_q, inv_collector, block_handler, block_chain,
                     block_store))
    return block_handler, block_chain, block_store, ap
def test_simple_getheader():
    BLOCKS = make_blocks(20)
    blockchain1 = BlockChain()
    blockchain1.add_headers(BLOCKS)

    block_store = dict((b.hash(), b) for b in BLOCKS)

    peer1, peer2 = create_handshaked_peers()

    block_store = {}
    block_chain = BlockChain()
    inv_collector = InvCollector()
    block_handler = BlockHandler(inv_collector, block_chain, block_store)

    for block in BLOCKS:
        inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
        block_store[block.hash()] = block
    block_chain.add_headers(BLOCKS)

    inv_collector.add_peer(peer1)
    block_handler.add_peer(peer1)

    @asyncio.coroutine
    def run_peer2():
        r = []
        headers = yield from standards.get_headers_hashes(peer2, after_block_hash=b"\0" * 32)
        r.append(headers)
        return r

    f2 = asyncio.Task(run_peer2())

    asyncio.get_event_loop().run_until_complete(asyncio.wait([f2]))

    r = f2.result()
    assert len(r) == 1
    assert [b.hash() for b in r[0]] == [b.hash() for b in BLOCKS]
Example #10
0
 def run_local_peer(peer_list):
     inv_collector = InvCollector()
     for peer in peer_list:
         inv_collector.add_peer(peer)
     r = []
     inv_item_q = inv_collector.new_inv_item_queue()
     while len(r) < 10:
         inv_item = yield from inv_item_q.get()
         v = yield from inv_collector.fetch(inv_item)
         r.append(v)
     return r
Example #11
0
 def run_client(peer_list, block_list):
     block_chain = BlockChain()
     block_store = {}
     inv_collector = InvCollector()
     block_handler = BlockHandler(inv_collector, block_chain, block_store)
     for peer in peer_list:
         inv_collector.add_peer(peer)
         block_handler.add_peer(peer)
     for block in block_list:
         inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK,
                                              block.hash()))
         block_store[block.hash()] = block
     while len(block_store) < 2:
         yield from asyncio.sleep(0.1)
     return block_store
Example #12
0
 def run_local_peer(peer_list):
     inv_collector = InvCollector()
     for peer in peer_list:
         inv_collector.add_peer(peer)
     r = []
     inv_item_q = inv_collector.new_inv_item_queue()
     while len(r) < 5:
         yield from asyncio.sleep(0.1)
         inv_item = yield from inv_item_q.get()
         try:
             v = yield from asyncio.wait_for(inv_collector.fetch(inv_item), timeout=0.5)
             if v:
                 r.append(v)
         except asyncio.TimeoutError:
             pass
     return r
Example #13
0
def test_simple_getheader():
    BLOCKS = make_blocks(20)
    blockchain1 = BlockChain()
    blockchain1.add_headers(BLOCKS)

    block_store = dict((b.hash(), b) for b in BLOCKS)

    peer1, peer2 = create_handshaked_peers()

    block_store = {}
    block_chain = BlockChain()
    inv_collector = InvCollector()
    block_handler = BlockHandler(inv_collector, block_chain, block_store)

    for block in BLOCKS:
        inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
        block_store[block.hash()] = block
    block_chain.add_headers(BLOCKS)

    inv_collector.add_peer(peer1)
    block_handler.add_peer(peer1)

    @asyncio.coroutine
    def run_peer2():
        r = []
        headers = yield from standards.get_headers_hashes(
            peer2, after_block_hash=b'\0' * 32)
        r.append(headers)
        return r

    f2 = asyncio.Task(run_peer2())

    asyncio.get_event_loop().run_until_complete(asyncio.wait([f2]))

    r = f2.result()
    assert len(r) == 1
    assert [b.hash() for b in r[0]] == [b.hash() for b in BLOCKS]
Example #14
0
def main():
    parser = argparse.ArgumentParser(description="Watch Bitcoin network for new blocks.")
    parser.add_argument('-c', "--config-dir", help='The directory where config files are stored.')
    parser.add_argument(
        '-f', "--fast-forward", type=int,
        help="block index to fast-forward to (ie. don't download full blocks prior to this one)", default=0
    )
    parser.add_argument(
        '-d', "--depth", type=int,
        help="Minimum depth blocks must be buried before being dropped in blockdir", default=2
    )
    parser.add_argument( '-l', "--log-file", help="Path to log file", default=None)
    parser.add_argument("blockdir", help='The directory where new blocks are dropped.')

    asyncio.tasks._DEBUG = True
    logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
    logging.getLogger("asyncio").setLevel(logging.INFO)

    if 1:
        host_port_q = dns_bootstrap_host_port_q(MAINNET)
    else:
        host_port_q = asyncio.Queue()
        host_port_q.put_nowait(("127.0.0.1", 8333))

    args = parser.parse_args()

    if args.log_file:
        log_file(args.log_file)

    block_chain_store = BlockChainStore(args.config_dir)
    block_chain = BlockChain(did_lock_to_index_f=block_chain_store.did_lock_to_index)

    locker_task = block_chain_locker(block_chain)
    block_chain.add_nodes(block_chain_store.block_tuple_iterator())

    blockfetcher = Blockfetcher()
    inv_collector = InvCollector()

    block_store = TwoLevelDict()

    @asyncio.coroutine
    def _rotate(block_store):
        while True:
            block_store.rotate()
            yield from asyncio.sleep(1800)
    rotate_task = asyncio.Task(_rotate(block_store))

    blockhandler = BlockHandler(inv_collector, block_chain, block_store,
        should_download_f=lambda block_hash, block_index: block_index >= args.fast_forward)

    last_processed_block = max(get_last_processed_block(config_dir), args.fast_forward)
    update_last_processed_block(config_dir, last_processed_block)

    change_q = asyncio.Queue()
    from pycoinnet.util.BlockChain import _update_q
    block_chain.add_change_callback(lambda blockchain, ops: _update_q(change_q, ops))

    block_processor_task = asyncio.Task(
        block_processor(
            change_q, blockfetcher, args.config_dir, args.blockdir, args.depth)))

    fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)

    fetcher_task = asyncio.Task(new_block_fetcher(inv_collector, block_chain))

    def create_protocol_callback():
        peer = BitcoinPeerProtocol(MAINNET["MAGIC_HEADER"])
        install_pingpong_manager(peer)
        fetcher = Fetcher(peer)
        peer.add_task(run_peer(
            peer, fetcher, fast_forward_add_peer,
            blockfetcher, inv_collector, blockhandler))
        return peer

    connection_info_q = manage_connection_count(host_port_q, create_protocol_callback, 8)
    show_task = asyncio.Task(show_connection_info(connection_info_q))
    asyncio.get_event_loop().run_forever()
Example #15
0
    def __init__(self,
                 network,
                 host_port_q,
                 should_download_block_f,
                 block_chain_store,
                 blockchain_change_callback,
                 server_port=9999):
        """
        network:
            a value from pycoinnet.helpers.networks
        host_port_q:
            a Queue that is being fed potential places to connect
        should_download_block_f:
            a function that accepting(block_hash, block_index) and returning a boolean
            indicating whether that block should be downloaded. Only used during fast-forward.
        block_chain_store:
            usually a BlockChainStore instance
        blockchain_change_callback:
            a callback that expects (blockchain, list_of_ops) that is invoked whenever the
            block chain is updated; blockchain is a BlockChain object and list_of_ops is a pair
            of tuples of the form (op, block_hash, block_index) where op is one of "add" or "remove",
            block_hash is a binary block hash, and block_index is an integer index number.
        """

        block_chain = BlockChain(
            did_lock_to_index_f=block_chain_store.did_lock_to_index)

        block_chain.preload_locked_blocks(block_chain_store.headers())

        block_chain.add_change_callback(block_chain_locker_callback)

        self.blockfetcher = Blockfetcher()
        self.inv_collector = InvCollector()

        self.block_store = TwoLevelDict()

        @asyncio.coroutine
        def _rotate(block_store):
            while True:
                block_store.rotate()
                yield from asyncio.sleep(1800)

        self.rotate_task = asyncio.Task(_rotate(self.block_store))

        self.blockhandler = BlockHandler(
            self.inv_collector,
            block_chain,
            self.block_store,
            should_download_f=should_download_block_f)

        block_chain.add_change_callback(blockchain_change_callback)

        self.fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)
        self.fetcher_task = asyncio.Task(
            new_block_fetcher(self.inv_collector, block_chain))

        self.nonce = int.from_bytes(os.urandom(8), byteorder="big")
        self.subversion = "/Notoshi/".encode("utf8")

        @asyncio.coroutine
        def run_peer(peer, fetcher, fast_forward_add_peer, blockfetcher,
                     inv_collector, blockhandler):
            yield from asyncio.wait_for(peer.connection_made_future,
                                        timeout=None)
            version_parameters = version_data_for_peer(
                peer,
                local_port=(server_port or 0),
                last_block_index=block_chain.length(),
                nonce=self.nonce,
                subversion=self.subversion)
            version_data = yield from initial_handshake(
                peer, version_parameters)
            last_block_index = version_data["last_block_index"]
            fast_forward_add_peer(peer, last_block_index)
            blockfetcher.add_peer(peer, fetcher, last_block_index)
            inv_collector.add_peer(peer)
            blockhandler.add_peer(peer)

        def create_protocol_callback():
            peer = BitcoinPeerProtocol(network["MAGIC_HEADER"])
            install_pingpong_manager(peer)
            fetcher = Fetcher(peer)
            peer.add_task(
                run_peer(peer, fetcher, self.fast_forward_add_peer,
                         self.blockfetcher, self.inv_collector,
                         self.blockhandler))
            return peer

        self.connection_info_q = manage_connection_count(
            host_port_q, create_protocol_callback, 8)
        self.show_task = asyncio.Task(
            show_connection_info(self.connection_info_q))

        # listener
        @asyncio.coroutine
        def run_listener():
            abstract_server = None
            future_peer = asyncio.Future()
            try:
                abstract_server = yield from asyncio.get_event_loop(
                ).create_server(protocol_factory=create_protocol_callback,
                                port=server_port)
                return abstract_server
            except OSError as ex:
                logging.info("can't listen on port %d", server_port)

        if server_port:
            self.server_task = asyncio.Task(run_listener())
Example #16
0
    def __init__(self,
                 network,
                 initial_blockchain_view,
                 bloom_filter,
                 block_index_queue=None,
                 filter_f=lambda idx, h: True,
                 host_port_q=None):
        """
        network:
            a value from pycoinnet.helpers.networks
        initial_blockchain_view:
            BlockChainView instance which is update automatically
        bloom_filter:
            the filter sent to remotes
        block_index_queue:
            a Queue which is fed with (block, index) items which need to be processed
        host_port_q:
            a Queue that is being fed potential places to connect
        """

        if host_port_q is None:
            host_port_q = dns_bootstrap_host_port_q(network)

        self.network = network
        self.blockchain_view = initial_blockchain_view
        self.bloom_filter = bloom_filter

        self.block_futures = asyncio.Queue(maxsize=2000)
        if block_index_queue is None:
            block_index_queue = asyncio.Queue()
        self._block_index_queue = block_index_queue
        self.feed_task = asyncio.Task(self.feed_blocks())

        self.blockfetcher = Blockfetcher()
        self.inv_collector = InvCollector()

        if bloom_filter:
            self.get_future = self.blockfetcher.get_merkle_block_future
        else:
            self.get_future = self.blockfetcher.get_block_future

        self.filter_f = filter_f

        self.getheaders_add_peer = getheaders_add_peer_f(
            self.blockchain_view, self.handle_reorg)

        self.nonce = int.from_bytes(os.urandom(8), byteorder="big")
        self.subversion = "/Notoshi/".encode("utf8")

        @asyncio.coroutine
        def run_peer(peer, fetcher, getheaders_add_peer, blockfetcher,
                     inv_collector):
            yield from asyncio.wait_for(peer.connection_made_future,
                                        timeout=None)
            last_block_index = max(0, self.blockchain_view.last_block_index())
            version_parameters = version_data_for_peer(
                peer,
                local_port=0,
                last_block_index=last_block_index,
                nonce=self.nonce,
                subversion=self.subversion)
            version_data = yield from initial_handshake(
                peer, version_parameters)
            if self.bloom_filter:
                filter_bytes, hash_function_count, tweak = self.bloom_filter.filter_load_params(
                )
                # TODO: figure out flags
                flags = 0
                peer.send_msg("filterload",
                              filter=filter_bytes,
                              hash_function_count=hash_function_count,
                              tweak=tweak,
                              flags=flags)
            last_block_index = version_data["last_block_index"]
            getheaders_add_peer(peer, last_block_index)
            blockfetcher.add_peer(peer, fetcher, last_block_index)
            inv_collector.add_peer(peer)

        def create_protocol_callback():
            peer = BitcoinPeerProtocol(network["MAGIC_HEADER"])
            install_pingpong_manager(peer)
            fetcher = Fetcher(peer)
            peer.add_task(
                run_peer(peer, fetcher, self.getheaders_add_peer,
                         self.blockfetcher, self.inv_collector))
            return peer

        self.connection_info_q = manage_connection_count(
            host_port_q, create_protocol_callback, 8)
        self.show_task = asyncio.Task(
            show_connection_info(self.connection_info_q))