def run_client(peer_list, tx_list):
tx_store = {}
inv_collector = InvCollector()
tx_handler = TxHandler(inv_collector, tx_store)
for tx in tx_list:
tx_handler.add_tx(tx)
for peer in peer_list:
inv_collector.add_peer(peer)
tx_handler.add_peer(peer)
while len(tx_store) < 20:
yield from asyncio.sleep(0.1)
return tx_store
def run_client(peer_list, tx_list):
tx_store = {}
inv_collector = InvCollector()
tx_handler = TxHandler(inv_collector, tx_store)
for tx in tx_list:
tx_handler.add_tx(tx)
for peer in peer_list:
inv_collector.add_peer(peer)
tx_handler.add_peer(peer)
while len(tx_store) < 20:
yield from asyncio.sleep(0.1)
return tx_store
def test_InvCollector():
# create some peers
peer1_2, peer2 = create_handshaked_peers()
peer1_3, peer3 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.3")
peer1_4, peer4 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.4")
# peer1_* represents the local peer
TX_LIST = [make_tx(i) for i in range(100)]
@asyncio.coroutine
def run_local_peer(inv_collector, inv_item_q):
r = []
while len(r) < 90:
inv_item = yield from inv_item_q.get()
v = yield from inv_collector.fetch(inv_item)
r.append(v)
return r
@asyncio.coroutine
def run_remote_peer(next_message, peer, txs):
tx_db = dict((tx.hash(), tx) for tx in txs)
r = []
inv_items = [InvItem(ITEM_TYPE_TX, tx.hash()) for tx in txs]
peer.send_msg("inv", items=inv_items)
while True:
t = yield from next_message()
r.append(t)
if t[0] == 'getdata':
for inv_item in t[1]["items"]:
peer.send_msg("tx", tx=tx_db[inv_item.data])
return r
futures = []
for peer, txs in [(peer2, TX_LIST[:30]), (peer3, TX_LIST[30:60]),
(peer4, TX_LIST[60:90])]:
f = asyncio.Task(
run_remote_peer(peer.new_get_next_message_f(), peer, txs))
futures.append(f)
inv_collector = InvCollector()
[inv_collector.add_peer(peer) for peer in [peer1_2, peer1_3, peer1_4]]
f = asyncio.Task(
run_local_peer(inv_collector, inv_collector.new_inv_item_queue()))
done, pending = asyncio.get_event_loop().run_until_complete(
asyncio.wait([f], timeout=5.0))
r = done.pop().result()
assert len(r) == 90
assert [tx.hash() for tx in r] == [tx.hash() for tx in TX_LIST[:90]]
def run_client(peer_list, block_list):
block_chain = BlockChain()
block_store = {}
inv_collector = InvCollector()
block_handler = BlockHandler(inv_collector, block_chain, block_store)
for peer in peer_list:
inv_collector.add_peer(peer)
block_handler.add_peer(peer)
for block in block_list:
inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
block_store[block.hash()] = block
while len(block_store) < 2:
yield from asyncio.sleep(0.1)
return block_store
def test_InvCollector():
# create some peers
peer1_2, peer2 = create_handshaked_peers()
peer1_3, peer3 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.3")
peer1_4, peer4 = create_handshaked_peers(ip1="127.0.0.1", ip2="127.0.0.4")
# peer1_* represents the local peer
TX_LIST = [make_tx(i) for i in range(100)]
@asyncio.coroutine
def run_local_peer(inv_collector, inv_item_q):
r = []
while len(r) < 90:
inv_item = yield from inv_item_q.get()
v = yield from inv_collector.fetch(inv_item)
r.append(v)
return r
@asyncio.coroutine
def run_remote_peer(next_message, peer, txs):
tx_db = dict((tx.hash(), tx) for tx in txs)
r = []
inv_items = [InvItem(ITEM_TYPE_TX, tx.hash()) for tx in txs]
peer.send_msg("inv", items=inv_items)
while True:
t = yield from next_message()
r.append(t)
if t[0] == 'getdata':
for inv_item in t[1]["items"]:
peer.send_msg("tx", tx=tx_db[inv_item.data])
return r
futures = []
for peer, txs in [(peer2, TX_LIST[:30]), (peer3, TX_LIST[30:60]), (peer4, TX_LIST[60:90])]:
f = asyncio.Task(run_remote_peer(peer.new_get_next_message_f(), peer, txs))
futures.append(f)
inv_collector = InvCollector()
[inv_collector.add_peer(peer) for peer in [peer1_2, peer1_3, peer1_4]]
f = asyncio.Task(run_local_peer(inv_collector, inv_collector.new_inv_item_queue()))
done, pending = asyncio.get_event_loop().run_until_complete(asyncio.wait([f], timeout=5.0))
r = done.pop().result()
assert len(r) == 90
assert [tx.hash() for tx in r] == [tx.hash() for tx in TX_LIST[:90]]
def items_for_client(initial_blocks=[]):
block_store = {}
block_chain = BlockChain()
blockfetcher = Blockfetcher()
inv_collector = InvCollector()
block_handler = BlockHandler(inv_collector, block_chain, block_store)
fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)
for block in initial_blocks:
inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
block_store[block.hash()] = block
block_chain.add_headers(initial_blocks)
inv_q = inv_collector.new_inv_item_queue()
ap = make_add_peer(fast_forward_add_peer, blockfetcher, block_handler, inv_collector, block_chain, block_store)
ap.block_getter_task = asyncio.Task(block_getter(inv_q, inv_collector, block_handler, block_chain, block_store))
return block_handler, block_chain, block_store, ap
def run_local_peer(peer_list, r):
inv_collector = InvCollector()
for peer in peer_list:
inv_collector.add_peer(peer)
inv_item_q = inv_collector.new_inv_item_queue()
@asyncio.coroutine
def _do_fetch(inv_collector, inv_item, r):
v = yield from inv_collector.fetch(inv_item, peer_timeout=3.0)
if v:
r.append(v)
# keep a strong reference to these tasks
tasks = []
for i in range(10):
inv_item = yield from inv_item_q.get()
tasks.append(asyncio.Task(_do_fetch(inv_collector, inv_item, r)))
while len(r) < 10:
yield from asyncio.sleep(0.1)
return r
def items_for_client(initial_blocks=[]):
block_store = {}
block_chain = BlockChain()
blockfetcher = Blockfetcher()
inv_collector = InvCollector()
block_handler = BlockHandler(inv_collector, block_chain, block_store)
fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)
for block in initial_blocks:
inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
block_store[block.hash()] = block
block_chain.add_headers(initial_blocks)
inv_q = inv_collector.new_inv_item_queue()
ap = make_add_peer(fast_forward_add_peer, blockfetcher, block_handler,
inv_collector, block_chain, block_store)
ap.block_getter_task = asyncio.Task(
block_getter(inv_q, inv_collector, block_handler, block_chain,
block_store))
return block_handler, block_chain, block_store, ap
def test_simple_getheader():
BLOCKS = make_blocks(20)
blockchain1 = BlockChain()
blockchain1.add_headers(BLOCKS)
block_store = dict((b.hash(), b) for b in BLOCKS)
peer1, peer2 = create_handshaked_peers()
block_store = {}
block_chain = BlockChain()
inv_collector = InvCollector()
block_handler = BlockHandler(inv_collector, block_chain, block_store)
for block in BLOCKS:
inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
block_store[block.hash()] = block
block_chain.add_headers(BLOCKS)
inv_collector.add_peer(peer1)
block_handler.add_peer(peer1)
@asyncio.coroutine
def run_peer2():
r = []
headers = yield from standards.get_headers_hashes(peer2, after_block_hash=b"\0" * 32)
r.append(headers)
return r
f2 = asyncio.Task(run_peer2())
asyncio.get_event_loop().run_until_complete(asyncio.wait([f2]))
r = f2.result()
assert len(r) == 1
assert [b.hash() for b in r[0]] == [b.hash() for b in BLOCKS]
def run_local_peer(peer_list):
inv_collector = InvCollector()
for peer in peer_list:
inv_collector.add_peer(peer)
r = []
inv_item_q = inv_collector.new_inv_item_queue()
while len(r) < 10:
inv_item = yield from inv_item_q.get()
v = yield from inv_collector.fetch(inv_item)
r.append(v)
return r
def run_client(peer_list, block_list):
block_chain = BlockChain()
block_store = {}
inv_collector = InvCollector()
block_handler = BlockHandler(inv_collector, block_chain, block_store)
for peer in peer_list:
inv_collector.add_peer(peer)
block_handler.add_peer(peer)
for block in block_list:
inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK,
block.hash()))
block_store[block.hash()] = block
while len(block_store) < 2:
yield from asyncio.sleep(0.1)
return block_store
def run_local_peer(peer_list):
inv_collector = InvCollector()
for peer in peer_list:
inv_collector.add_peer(peer)
r = []
inv_item_q = inv_collector.new_inv_item_queue()
while len(r) < 5:
yield from asyncio.sleep(0.1)
inv_item = yield from inv_item_q.get()
try:
v = yield from asyncio.wait_for(inv_collector.fetch(inv_item), timeout=0.5)
if v:
r.append(v)
except asyncio.TimeoutError:
pass
return r
def test_simple_getheader():
BLOCKS = make_blocks(20)
blockchain1 = BlockChain()
blockchain1.add_headers(BLOCKS)
block_store = dict((b.hash(), b) for b in BLOCKS)
peer1, peer2 = create_handshaked_peers()
block_store = {}
block_chain = BlockChain()
inv_collector = InvCollector()
block_handler = BlockHandler(inv_collector, block_chain, block_store)
for block in BLOCKS:
inv_collector.advertise_item(InvItem(ITEM_TYPE_BLOCK, block.hash()))
block_store[block.hash()] = block
block_chain.add_headers(BLOCKS)
inv_collector.add_peer(peer1)
block_handler.add_peer(peer1)
@asyncio.coroutine
def run_peer2():
r = []
headers = yield from standards.get_headers_hashes(
peer2, after_block_hash=b'\0' * 32)
r.append(headers)
return r
f2 = asyncio.Task(run_peer2())
asyncio.get_event_loop().run_until_complete(asyncio.wait([f2]))
r = f2.result()
assert len(r) == 1
assert [b.hash() for b in r[0]] == [b.hash() for b in BLOCKS]
def main():
parser = argparse.ArgumentParser(description="Watch Bitcoin network for new blocks.")
parser.add_argument('-c', "--config-dir", help='The directory where config files are stored.')
parser.add_argument(
'-f', "--fast-forward", type=int,
help="block index to fast-forward to (ie. don't download full blocks prior to this one)", default=0
)
parser.add_argument(
'-d', "--depth", type=int,
help="Minimum depth blocks must be buried before being dropped in blockdir", default=2
)
parser.add_argument( '-l', "--log-file", help="Path to log file", default=None)
parser.add_argument("blockdir", help='The directory where new blocks are dropped.')
asyncio.tasks._DEBUG = True
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
logging.getLogger("asyncio").setLevel(logging.INFO)
if 1:
host_port_q = dns_bootstrap_host_port_q(MAINNET)
else:
host_port_q = asyncio.Queue()
host_port_q.put_nowait(("127.0.0.1", 8333))
args = parser.parse_args()
if args.log_file:
log_file(args.log_file)
block_chain_store = BlockChainStore(args.config_dir)
block_chain = BlockChain(did_lock_to_index_f=block_chain_store.did_lock_to_index)
locker_task = block_chain_locker(block_chain)
block_chain.add_nodes(block_chain_store.block_tuple_iterator())
blockfetcher = Blockfetcher()
inv_collector = InvCollector()
block_store = TwoLevelDict()
@asyncio.coroutine
def _rotate(block_store):
while True:
block_store.rotate()
yield from asyncio.sleep(1800)
rotate_task = asyncio.Task(_rotate(block_store))
blockhandler = BlockHandler(inv_collector, block_chain, block_store,
should_download_f=lambda block_hash, block_index: block_index >= args.fast_forward)
last_processed_block = max(get_last_processed_block(config_dir), args.fast_forward)
update_last_processed_block(config_dir, last_processed_block)
change_q = asyncio.Queue()
from pycoinnet.util.BlockChain import _update_q
block_chain.add_change_callback(lambda blockchain, ops: _update_q(change_q, ops))
block_processor_task = asyncio.Task(
block_processor(
change_q, blockfetcher, args.config_dir, args.blockdir, args.depth)))
fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)
fetcher_task = asyncio.Task(new_block_fetcher(inv_collector, block_chain))
def create_protocol_callback():
peer = BitcoinPeerProtocol(MAINNET["MAGIC_HEADER"])
install_pingpong_manager(peer)
fetcher = Fetcher(peer)
peer.add_task(run_peer(
peer, fetcher, fast_forward_add_peer,
blockfetcher, inv_collector, blockhandler))
return peer
connection_info_q = manage_connection_count(host_port_q, create_protocol_callback, 8)
show_task = asyncio.Task(show_connection_info(connection_info_q))
asyncio.get_event_loop().run_forever()
def __init__(self,
network,
host_port_q,
should_download_block_f,
block_chain_store,
blockchain_change_callback,
server_port=9999):
"""
network:
a value from pycoinnet.helpers.networks
host_port_q:
a Queue that is being fed potential places to connect
should_download_block_f:
a function that accepting(block_hash, block_index) and returning a boolean
indicating whether that block should be downloaded. Only used during fast-forward.
block_chain_store:
usually a BlockChainStore instance
blockchain_change_callback:
a callback that expects (blockchain, list_of_ops) that is invoked whenever the
block chain is updated; blockchain is a BlockChain object and list_of_ops is a pair
of tuples of the form (op, block_hash, block_index) where op is one of "add" or "remove",
block_hash is a binary block hash, and block_index is an integer index number.
"""
block_chain = BlockChain(
did_lock_to_index_f=block_chain_store.did_lock_to_index)
block_chain.preload_locked_blocks(block_chain_store.headers())
block_chain.add_change_callback(block_chain_locker_callback)
self.blockfetcher = Blockfetcher()
self.inv_collector = InvCollector()
self.block_store = TwoLevelDict()
@asyncio.coroutine
def _rotate(block_store):
while True:
block_store.rotate()
yield from asyncio.sleep(1800)
self.rotate_task = asyncio.Task(_rotate(self.block_store))
self.blockhandler = BlockHandler(
self.inv_collector,
block_chain,
self.block_store,
should_download_f=should_download_block_f)
block_chain.add_change_callback(blockchain_change_callback)
self.fast_forward_add_peer = fast_forwarder_add_peer_f(block_chain)
self.fetcher_task = asyncio.Task(
new_block_fetcher(self.inv_collector, block_chain))
self.nonce = int.from_bytes(os.urandom(8), byteorder="big")
self.subversion = "/Notoshi/".encode("utf8")
@asyncio.coroutine
def run_peer(peer, fetcher, fast_forward_add_peer, blockfetcher,
inv_collector, blockhandler):
yield from asyncio.wait_for(peer.connection_made_future,
timeout=None)
version_parameters = version_data_for_peer(
peer,
local_port=(server_port or 0),
last_block_index=block_chain.length(),
nonce=self.nonce,
subversion=self.subversion)
version_data = yield from initial_handshake(
peer, version_parameters)
last_block_index = version_data["last_block_index"]
fast_forward_add_peer(peer, last_block_index)
blockfetcher.add_peer(peer, fetcher, last_block_index)
inv_collector.add_peer(peer)
blockhandler.add_peer(peer)
def create_protocol_callback():
peer = BitcoinPeerProtocol(network["MAGIC_HEADER"])
install_pingpong_manager(peer)
fetcher = Fetcher(peer)
peer.add_task(
run_peer(peer, fetcher, self.fast_forward_add_peer,
self.blockfetcher, self.inv_collector,
self.blockhandler))
return peer
self.connection_info_q = manage_connection_count(
host_port_q, create_protocol_callback, 8)
self.show_task = asyncio.Task(
show_connection_info(self.connection_info_q))
# listener
@asyncio.coroutine
def run_listener():
abstract_server = None
future_peer = asyncio.Future()
try:
abstract_server = yield from asyncio.get_event_loop(
).create_server(protocol_factory=create_protocol_callback,
port=server_port)
return abstract_server
except OSError as ex:
logging.info("can't listen on port %d", server_port)
if server_port:
self.server_task = asyncio.Task(run_listener())
def __init__(self,
network,
initial_blockchain_view,
bloom_filter,
block_index_queue=None,
filter_f=lambda idx, h: True,
host_port_q=None):
"""
network:
a value from pycoinnet.helpers.networks
initial_blockchain_view:
BlockChainView instance which is update automatically
bloom_filter:
the filter sent to remotes
block_index_queue:
a Queue which is fed with (block, index) items which need to be processed
host_port_q:
a Queue that is being fed potential places to connect
"""
if host_port_q is None:
host_port_q = dns_bootstrap_host_port_q(network)
self.network = network
self.blockchain_view = initial_blockchain_view
self.bloom_filter = bloom_filter
self.block_futures = asyncio.Queue(maxsize=2000)
if block_index_queue is None:
block_index_queue = asyncio.Queue()
self._block_index_queue = block_index_queue
self.feed_task = asyncio.Task(self.feed_blocks())
self.blockfetcher = Blockfetcher()
self.inv_collector = InvCollector()
if bloom_filter:
self.get_future = self.blockfetcher.get_merkle_block_future
else:
self.get_future = self.blockfetcher.get_block_future
self.filter_f = filter_f
self.getheaders_add_peer = getheaders_add_peer_f(
self.blockchain_view, self.handle_reorg)
self.nonce = int.from_bytes(os.urandom(8), byteorder="big")
self.subversion = "/Notoshi/".encode("utf8")
@asyncio.coroutine
def run_peer(peer, fetcher, getheaders_add_peer, blockfetcher,
inv_collector):
yield from asyncio.wait_for(peer.connection_made_future,
timeout=None)
last_block_index = max(0, self.blockchain_view.last_block_index())
version_parameters = version_data_for_peer(
peer,
local_port=0,
last_block_index=last_block_index,
nonce=self.nonce,
subversion=self.subversion)
version_data = yield from initial_handshake(
peer, version_parameters)
if self.bloom_filter:
filter_bytes, hash_function_count, tweak = self.bloom_filter.filter_load_params(
)
# TODO: figure out flags
flags = 0
peer.send_msg("filterload",
filter=filter_bytes,
hash_function_count=hash_function_count,
tweak=tweak,
flags=flags)
last_block_index = version_data["last_block_index"]
getheaders_add_peer(peer, last_block_index)
blockfetcher.add_peer(peer, fetcher, last_block_index)
inv_collector.add_peer(peer)
def create_protocol_callback():
peer = BitcoinPeerProtocol(network["MAGIC_HEADER"])
install_pingpong_manager(peer)
fetcher = Fetcher(peer)
peer.add_task(
run_peer(peer, fetcher, self.getheaders_add_peer,
self.blockfetcher, self.inv_collector))
return peer
self.connection_info_q = manage_connection_count(
host_port_q, create_protocol_callback, 8)
self.show_task = asyncio.Task(
show_connection_info(self.connection_info_q))