async def send_it(self, txn_dict: dict, peer: Peer): try: if self.config.debug: self.app_log.debug( 'Transmitting pool payout transaction to: {}'.format( peer.to_string())) await peer.client.client.emit('newtransaction', data=txn_dict, namespace='/chat') except Exception as e: if self.config.debug: self.app_log.debug(e)
async def post(self): """ A peer does notify us of a new block. This is deprecated, since the new code uses events via websocket to notify of a new block. Still, can be used to force notification to important nodes, pools... """ from yadacoin.peers import Peer try: block_data = escape.json_decode(self.request.body) peer_string = block_data.get('peer') if block_data['index'] == 0: return if int(block_data['version'] ) != self.config.BU.get_version_for_height( block_data['index']): print('rejected old version %s from %s' % (block_data['version'], peer_string)) return # Dup code with websocket handler self.app_log.info('Post new block: {} {}'.format( peer_string, json.dumps(block_data))) # TODO: handle a dict here to store the consensus state if not self.peers.syncing: self.app_log.debug( "Trying to sync on latest block from {}".format( peer_string)) my_index = self.config.LatestBlock.block.index # This is mostly to keep in sync with fast moving blocks from whitelisted peers and pools. # ignore if this does not fit. if block_data['index'] == my_index + 1: self.app_log.debug( "Next index, trying to merge from {}".format( peer_string)) peer = Peer.from_string(peer_string) if await self.config.consensus.process_next_block( block_data, peer): pass # if ok, block was inserted and event triggered by import block # await self.peers.on_block_insert(data) elif block_data['index'] > my_index + 1: self.app_log.warning( "Missing blocks between {} and {} , can't catch up from http route for {}" .format(my_index, block_data['index'], peer_string)) # data = {"start_index": my_index + 1, "end_index": my_index + 1 + CHAIN.MAX_BLOCKS_PER_MESSAGE} # await self.emit('get_blocks', data=data, room=sid) else: # Remove later on self.app_log.debug( "Old or same index, ignoring {} from {}".format( block_data['index'], peer_string)) except: print('ERROR: failed to get peers, exiting...')
async def on_latest_block(self, sid, data): """Client informs us of its new block""" # from yadacoin.block import Block # Circular reference. Not good! - Do we need the object here? self.app_log.info('WS latest-block: {} {}'.format( sid, json.dumps(data))) # TODO: handle a dict here to store the consensus state # self.latest_peer_block = Block.from_dict(data) if not self.peers.syncing: async with self.session(sid) as session: peer = Peer(session['ip'], session['port']) self.app_log.debug("Trying to sync on latest block from {}".format( peer.to_string())) my_index = self.config.BU.get_latest_block()['index'] if data['index'] == my_index + 1: self.app_log.debug( "Next index, trying to merge from {}".format( peer.to_string())) if await self.consensus.process_next_block(data, peer): pass # if ok, block was inserted and event triggered by import block # await self.peers.on_block_insert(data) elif data['index'] > my_index + 1: self.app_log.debug( "Missing blocks between {} and {} , asking more to {}". format(my_index, data['index'], peer.to_string())) data = { "start_index": my_index + 1, "end_index": my_index + 1 + CHAIN.MAX_BLOCKS_PER_MESSAGE } await self.emit('get_blocks', data=data, room=sid) else: # Remove later on self.app_log.debug( "Old or same index, ignoring {} from {}".format( data['index'], peer.to_string()))
def rank_consensus_blocks(self): # rank is based on target, total chain difficulty, and chain validity records = self.get_consensus_blocks_by_index(self.latest_block.index + 1) lowest = self.lowest ranks = [] for record in records: peer = Peer.from_string(record['peer']) block = Block.from_dict(record['block']) target = int(record['block']['hash'], 16) if target < lowest: ranks.append({'target': target, 'block': block, 'peer': peer}) return sorted(ranks, key=lambda x: x['target'])
async def txn_broadcast_job(self, transaction): if self.config.network != 'regnet': for peer in self.config.peers.peers: if not isinstance(peer, Peer): peer = Peer(peer['host'], peer['port']) if peer.host in self.config.outgoing_blacklist or not ( peer.client and peer.client.connected): continue if peer.host == self.config.peer_host and peer.port == self.config.peer_port: continue try: # peer = self.config.peers.my_peer await self.send_it(transaction.to_dict(), peer) except Exception as e: print("Error ", e)
async def on_blocks(self, sid, data): self.app_log.info('WS blocks: {} {}'.format(sid, json.dumps(data))) if not len(data): return my_index = self.config.BU.get_latest_block()['index'] if data[0]['index'] != my_index + 1: return self.peers.syncing = True try: async with self.session(sid) as session: peer = Peer(session['ip'], session['port']) inserted = False block = None # Avoid linter warning for block in data: # print("looking for ", self.existing_blockchain.blocks[-1].index + 1) if block['index'] == my_index + 1: if await self.consensus.process_next_block( block, peer, trigger_event=False): inserted = True my_index = block['index'] else: break else: # As soon as a block fails, abort break if inserted: # If import was successful, inform out peers once the batch is processed await self.peers.on_block_insert(block) # then ask for the potential next batch data = { "start_index": my_index + 1, "end_index": my_index + 1 + CHAIN.MAX_BLOCKS_PER_MESSAGE } await self.emit('get_blocks', data=data, room=sid) else: self.app_log.debug("Import aborted block: {}".format(my_index)) return except Exception as e: import sys, os self.app_log.warning("Exception {} on_blocks".format(e)) exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print(exc_type, fname, exc_tb.tb_lineno) finally: self.peers.syncing = False
async def main(): global config define("debug", default=False, help="debug mode", type=bool) define("verbose", default=False, help="verbose mode", type=bool) define("network", default='', help="Force mainnet, testnet or regnet", type=str) define("reset", default=False, help="If blockchain is invalid, truncate at error block", type=bool) define("config", default='config/config.json', help="Config file location, default is 'config/config.json'", type=str) define("verify", default=True, help="Verify chain, default True", type=bool) define( "webonly", default=False, help= "Web only (ignores node processes for faster init when restarting server frequently), default False", type=bool) options.parse_command_line(final=False) configure_logging() if not path.isfile(options.config): app_log.error("no config file found at '{}'".format(options.config)) exit() with open(options.config) as f: config = yadacoin.config.Config(json.loads(f.read())) # Sets the global var for all objects yadacoin.config.CONFIG = config config.debug = options.debug # force network, command line one takes precedence if options.network != '': config.network = options.network config.protocol_version = PROTOCOL_VERSION # get seed.json from same dir as config. if config.network == 'mainnet': seed_filename = 'seed.json' elif config.network == 'testnet': seed_filename = 'seed_testnet.json' peers_seed_filename = options.config.replace( ntpath.basename(options.config), seed_filename) if path.isfile(peers_seed_filename): with open(peers_seed_filename) as f: config.peers_seed = json.loads(f.read()) mongo = Mongo() config.mongo = mongo peers = Peers() config.peers = peers if not options.webonly: config.BU = yadacoin.blockchainutils.BlockChainUtils() config.TU = yadacoin.transactionutils.TU yadacoin.blockchainutils.set_BU(config.BU) # To be removed config.GU = GraphUtils() consensus = Consensus(options.debug, peers) if options.verify: app_log.info("Verifying existing blockchain") consensus.verify_existing_blockchain(reset=options.reset) else: app_log.info( "Verification of existing blockchain skipped by config") config.consensus = consensus if config.max_miners > 0: app_log.info("MiningPool activated, max miners {}".format( config.max_miners)) else: app_log.info("MiningPool disabled by config") ws_init() config.SIO = get_sio() tornado.ioloop.IOLoop.instance().add_callback(background_consensus, consensus) tornado.ioloop.IOLoop.instance().add_callback(background_peers, peers) tornado.ioloop.IOLoop.instance().add_callback(background_status) tornado.ioloop.IOLoop.instance().add_callback(background_pool) if config.pool_payout: app_log.info("PoolPayout activated") pp = PoolPayer() config.pp = pp tornado.ioloop.IOLoop.instance().add_callback( background_pool_payer) my_peer = Peer.init_my_peer(config.network) app_log.info("API: http://{}".format(my_peer.to_string())) app = NodeApplication(config, mongo, peers) app_log.info("Starting server on {}:{}".format(config.serve_host, config.serve_port)) app.listen(config.serve_port, config.serve_host) if config.ssl: ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=config.ssl.get('cafile')) ssl_ctx.load_cert_chain(config.ssl.get('certfile'), keyfile=config.ssl.get('keyfile')) http_server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_ctx) http_server.listen(config.ssl['port']) # The server will simply run until interrupted # with Ctrl-C, but if you want to shut down more gracefully, # call shutdown_event.set(). shutdown_event = tornado.locks.Event() await shutdown_event.wait()
async def import_block(self, block_data: dict, trigger_event=True) -> bool: """Block_data contains peer and block keys. Tries to import that block, retrace if necessary sends True if that block was inserted, False if it fails or if a retrace was needed. This is the central entry point for inserting a block, that will modify the local chain and trigger the event, unless we asked not to, because we're in a batch insert context""" try: block = Block.from_dict(block_data['block']) peer = Peer.from_string(block_data['peer']) if 'extra_blocks' in block_data: extra_blocks = None # extra_blocks = [Block.from_dict( x) for x in block_data['extra_blocks']] # Not used later on, just ram and resources usage else: extra_blocks = None self.app_log.debug("Latest block was {} {} {} {}".format( self.latest_block.hash, block.prev_hash, self.latest_block.index, (block.index - 1))) if int(block.index) > CHAIN.CHECK_TIME_FROM and int( block.time) < int(self.latest_block.time): self.app_log.warning( "New block {} can't be at a sooner time than previous one. Rejecting" .format(block.index)) await self.mongo.async_db.consensus.update_one( { 'peer': peer.to_string(), 'index': block.index, 'id': block.signature }, {'$set': { 'ignore': True }}) return False if int(block.index) > CHAIN.CHECK_TIME_FROM and ( int(block.time) < (int(self.latest_block.time) + 600)) and block.special_min: self.app_log.warning( "New special min block {} too soon. Rejecting".format( block.index)) await self.mongo.async_db.consensus.update_one( { 'peer': peer.to_string(), 'index': block.index, 'id': block.signature }, {'$set': { 'ignore': True }}) return False try: result = await self.integrate_block_with_existing_chain( block, extra_blocks) if result is False: # TODO: factorize await self.mongo.async_db.consensus.update_one( { 'peer': peer.to_string(), 'index': block.index, 'id': block.signature }, {'$set': { 'ignore': True }}) elif trigger_event: await self.trigger_update_event(block_data['block']) return result except DuplicateKeyError as e: await self.mongo.async_db.consensus.update_one( { 'peer': peer.to_string(), 'index': block.index, 'id': block.signature }, {'$set': { 'ignore': True }}) except AboveTargetException as e: await self.mongo.async_db.consensus.update_one( { 'peer': peer.to_string(), 'index': block.index, 'id': block.signature }, {'$set': { 'ignore': True }}) except ForkException as e: await self.retrace(block, peer) if trigger_event: await self.trigger_update_event() return False except IndexError as e: await self.retrace(block, peer) if trigger_event: await self.trigger_update_event() return False except Exception as e: print("348", e) exc_type, exc_obj, exc_tb = exc_info() fname = path.split(exc_tb.tb_frame.f_code.co_filename)[1] self.app_log.warning("{} {} {}".format(exc_type, fname, exc_tb.tb_lineno)) await self.mongo.async_db.consensus.update_one( { 'peer': peer.to_string(), 'index': block.index, 'id': block.signature }, {'$set': { 'ignore': True }}) except Exception as e: exc_type, exc_obj, exc_tb = exc_info() fname = path.split(exc_tb.tb_frame.f_code.co_filename)[1] self.app_log.warning("{} {} {}".format(exc_type, fname, exc_tb.tb_lineno)) if trigger_event: await self.trigger_update_event() return False if trigger_event: await self.trigger_update_event() return True
async def search_network_for_new(self): # Peers.init( self.config.network) if self.config.network == 'regnet': return if self.peers.syncing: self.app_log.debug( "Already syncing, ignoring search_network_for_new") if len(self.config.force_polling): # This is a temp hack until everyone updated polling_peers = [ "{}:{}".format(peer['host'], peer['port']) for peer in self.config.force_polling ] else: if len(self.peers.peers) < 2: await self.peers.refresh() await async_sleep(20) if len(self.peers.peers) < 1: self.app_log.info("No peer to connect to yet") await async_sleep(10) return polling_peers = [peer.to_string() for peer in self.peers.peers] # TODO: use an aio lock self.app_log.debug('requesting {} ...'.format(self.latest_block.index + 1)) http_client = AsyncHTTPClient() # for peer in self.peers.peers: for peer_string in polling_peers: self.peers.syncing = True try: self.app_log.debug('requesting {} from {}'.format( self.latest_block.index + 1, peer_string)) peer = Peer.from_string(peer_string) try: url = 'http://{peer}/get-blocks?start_index={start_index}&end_index={end_index}'\ .format(peer=peer_string, start_index=int(self.latest_block.index) +1, end_index=int(self.latest_block.index) + 100) request = HTTPRequest(url, connect_timeout=3, request_timeout=5) response = await http_client.fetch(request) if response.code != 200: continue # result = requests.get(url, timeout=2) except HTTPError as e: self.app_log.warning( 'Error requesting from {} ...'.format(peer_string)) # add to failed peers await self.peers.increment_failed(peer) continue except ConnectTimeoutError as e: self.app_log.warning( 'Timeout requesting from {} ...'.format(peer_string)) # add to failed peers await self.peers.increment_failed(peer) continue except Exception as e: self.app_log.error( 'error {} requesting from {} ...'.format( e, peer_string)) await self.peers.increment_failed(peer) continue try: blocks = json.loads(response.body.decode('utf-8')) # blocks = json.loads(result.content) except ValueError: continue inserted = False for block in blocks: # print("looking for ", self.existing_blockchain.blocks[-1].index + 1) block = Block.from_dict(block) if block.index == ( self.existing_blockchain.blocks[-1].index + 1): await self.insert_consensus_block(block, peer) # print("consensus ok", block.index) res = await self.import_block( { 'peer': peer_string, 'block': block.to_dict(), 'extra_blocks': blocks }, trigger_event=False) # print("import ", block.index, res) if res: self.latest_block = block inserted = True else: # 2 cases: bad block, or retrace. if self.existing_blockchain.blocks[ -1].index == self.latest_block.index: # bad block, nothing moved, early exit self.app_log.debug('Bad block {}'.format( block.index)) else: # retraced, sync self.latest_block = Block.from_dict( await self.config.BU.get_latest_block_async()) self.app_log.debug('retraced up to {}'.format( self.latest_block.index)) inserted = True # in both case, no need to process further blocks break else: break #print("pass", block.index) if inserted: await self.trigger_update_event() # await self.peers.on_block_insert(self.latest_block.to_dict()) except Exception as e: if self.debug: self.app_log.warning(e) finally: self.peers.syncing = False
if not Peers.peers: time.sleep(1) continue Faucet.run(config, mongo) time.sleep(1) """ elif args.mode == 'pool': print("Not supported Yet") sys.exit() """ pp = PoolPayer(config, mongo) while 1: pp.do_payout() time.sleep(1) """ elif args.mode == 'serve': from yadacoin.peers import Peer from yadacoin.serve import Serve print(config.to_json()) config.network = args.network my_peer = Peer.init_my_peer(config, mongo, config.network) print("http://{}".format(my_peer.to_string())) app = Flask(__name__) serve = Serve(config, mongo, app) app.run(config.serve_host, config.serve_port)