async def send_it(self, txn_dict: dict, peer: Peer):
try:
if self.config.debug:
self.app_log.debug(
'Transmitting pool payout transaction to: {}'.format(
peer.to_string()))
await peer.client.client.emit('newtransaction',
data=txn_dict,
namespace='/chat')
except Exception as e:
if self.config.debug:
self.app_log.debug(e)
async def post(self):
"""
A peer does notify us of a new block. This is deprecated, since the new code uses events via websocket to notify of a new block.
Still, can be used to force notification to important nodes, pools...
"""
from yadacoin.peers import Peer
try:
block_data = escape.json_decode(self.request.body)
peer_string = block_data.get('peer')
if block_data['index'] == 0:
return
if int(block_data['version']
) != self.config.BU.get_version_for_height(
block_data['index']):
print('rejected old version %s from %s' %
(block_data['version'], peer_string))
return
# Dup code with websocket handler
self.app_log.info('Post new block: {} {}'.format(
peer_string, json.dumps(block_data)))
# TODO: handle a dict here to store the consensus state
if not self.peers.syncing:
self.app_log.debug(
"Trying to sync on latest block from {}".format(
peer_string))
my_index = self.config.LatestBlock.block.index
# This is mostly to keep in sync with fast moving blocks from whitelisted peers and pools.
# ignore if this does not fit.
if block_data['index'] == my_index + 1:
self.app_log.debug(
"Next index, trying to merge from {}".format(
peer_string))
peer = Peer.from_string(peer_string)
if await self.config.consensus.process_next_block(
block_data, peer):
pass
# if ok, block was inserted and event triggered by import block
# await self.peers.on_block_insert(data)
elif block_data['index'] > my_index + 1:
self.app_log.warning(
"Missing blocks between {} and {} , can't catch up from http route for {}"
.format(my_index, block_data['index'], peer_string))
# data = {"start_index": my_index + 1, "end_index": my_index + 1 + CHAIN.MAX_BLOCKS_PER_MESSAGE}
# await self.emit('get_blocks', data=data, room=sid)
else:
# Remove later on
self.app_log.debug(
"Old or same index, ignoring {} from {}".format(
block_data['index'], peer_string))
except:
print('ERROR: failed to get peers, exiting...')
async def on_latest_block(self, sid, data):
"""Client informs us of its new block"""
# from yadacoin.block import Block # Circular reference. Not good! - Do we need the object here?
self.app_log.info('WS latest-block: {} {}'.format(
sid, json.dumps(data)))
# TODO: handle a dict here to store the consensus state
# self.latest_peer_block = Block.from_dict(data)
if not self.peers.syncing:
async with self.session(sid) as session:
peer = Peer(session['ip'], session['port'])
self.app_log.debug("Trying to sync on latest block from {}".format(
peer.to_string()))
my_index = self.config.BU.get_latest_block()['index']
if data['index'] == my_index + 1:
self.app_log.debug(
"Next index, trying to merge from {}".format(
peer.to_string()))
if await self.consensus.process_next_block(data, peer):
pass
# if ok, block was inserted and event triggered by import block
# await self.peers.on_block_insert(data)
elif data['index'] > my_index + 1:
self.app_log.debug(
"Missing blocks between {} and {} , asking more to {}".
format(my_index, data['index'], peer.to_string()))
data = {
"start_index": my_index + 1,
"end_index": my_index + 1 + CHAIN.MAX_BLOCKS_PER_MESSAGE
}
await self.emit('get_blocks', data=data, room=sid)
else:
# Remove later on
self.app_log.debug(
"Old or same index, ignoring {} from {}".format(
data['index'], peer.to_string()))
def rank_consensus_blocks(self):
# rank is based on target, total chain difficulty, and chain validity
records = self.get_consensus_blocks_by_index(self.latest_block.index +
1)
lowest = self.lowest
ranks = []
for record in records:
peer = Peer.from_string(record['peer'])
block = Block.from_dict(record['block'])
target = int(record['block']['hash'], 16)
if target < lowest:
ranks.append({'target': target, 'block': block, 'peer': peer})
return sorted(ranks, key=lambda x: x['target'])
async def txn_broadcast_job(self, transaction):
if self.config.network != 'regnet':
for peer in self.config.peers.peers:
if not isinstance(peer, Peer):
peer = Peer(peer['host'], peer['port'])
if peer.host in self.config.outgoing_blacklist or not (
peer.client and peer.client.connected):
continue
if peer.host == self.config.peer_host and peer.port == self.config.peer_port:
continue
try:
# peer = self.config.peers.my_peer
await self.send_it(transaction.to_dict(), peer)
except Exception as e:
print("Error ", e)
async def on_blocks(self, sid, data):
self.app_log.info('WS blocks: {} {}'.format(sid, json.dumps(data)))
if not len(data):
return
my_index = self.config.BU.get_latest_block()['index']
if data[0]['index'] != my_index + 1:
return
self.peers.syncing = True
try:
async with self.session(sid) as session:
peer = Peer(session['ip'], session['port'])
inserted = False
block = None # Avoid linter warning
for block in data:
# print("looking for ", self.existing_blockchain.blocks[-1].index + 1)
if block['index'] == my_index + 1:
if await self.consensus.process_next_block(
block, peer, trigger_event=False):
inserted = True
my_index = block['index']
else:
break
else:
# As soon as a block fails, abort
break
if inserted:
# If import was successful, inform out peers once the batch is processed
await self.peers.on_block_insert(block)
# then ask for the potential next batch
data = {
"start_index": my_index + 1,
"end_index": my_index + 1 + CHAIN.MAX_BLOCKS_PER_MESSAGE
}
await self.emit('get_blocks', data=data, room=sid)
else:
self.app_log.debug("Import aborted block: {}".format(my_index))
return
except Exception as e:
import sys, os
self.app_log.warning("Exception {} on_blocks".format(e))
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
finally:
self.peers.syncing = False
async def main():
global config
define("debug", default=False, help="debug mode", type=bool)
define("verbose", default=False, help="verbose mode", type=bool)
define("network",
default='',
help="Force mainnet, testnet or regnet",
type=str)
define("reset",
default=False,
help="If blockchain is invalid, truncate at error block",
type=bool)
define("config",
default='config/config.json',
help="Config file location, default is 'config/config.json'",
type=str)
define("verify",
default=True,
help="Verify chain, default True",
type=bool)
define(
"webonly",
default=False,
help=
"Web only (ignores node processes for faster init when restarting server frequently), default False",
type=bool)
options.parse_command_line(final=False)
configure_logging()
if not path.isfile(options.config):
app_log.error("no config file found at '{}'".format(options.config))
exit()
with open(options.config) as f:
config = yadacoin.config.Config(json.loads(f.read()))
# Sets the global var for all objects
yadacoin.config.CONFIG = config
config.debug = options.debug
# force network, command line one takes precedence
if options.network != '':
config.network = options.network
config.protocol_version = PROTOCOL_VERSION
# get seed.json from same dir as config.
if config.network == 'mainnet':
seed_filename = 'seed.json'
elif config.network == 'testnet':
seed_filename = 'seed_testnet.json'
peers_seed_filename = options.config.replace(
ntpath.basename(options.config), seed_filename)
if path.isfile(peers_seed_filename):
with open(peers_seed_filename) as f:
config.peers_seed = json.loads(f.read())
mongo = Mongo()
config.mongo = mongo
peers = Peers()
config.peers = peers
if not options.webonly:
config.BU = yadacoin.blockchainutils.BlockChainUtils()
config.TU = yadacoin.transactionutils.TU
yadacoin.blockchainutils.set_BU(config.BU) # To be removed
config.GU = GraphUtils()
consensus = Consensus(options.debug, peers)
if options.verify:
app_log.info("Verifying existing blockchain")
consensus.verify_existing_blockchain(reset=options.reset)
else:
app_log.info(
"Verification of existing blockchain skipped by config")
config.consensus = consensus
if config.max_miners > 0:
app_log.info("MiningPool activated, max miners {}".format(
config.max_miners))
else:
app_log.info("MiningPool disabled by config")
ws_init()
config.SIO = get_sio()
tornado.ioloop.IOLoop.instance().add_callback(background_consensus,
consensus)
tornado.ioloop.IOLoop.instance().add_callback(background_peers, peers)
tornado.ioloop.IOLoop.instance().add_callback(background_status)
tornado.ioloop.IOLoop.instance().add_callback(background_pool)
if config.pool_payout:
app_log.info("PoolPayout activated")
pp = PoolPayer()
config.pp = pp
tornado.ioloop.IOLoop.instance().add_callback(
background_pool_payer)
my_peer = Peer.init_my_peer(config.network)
app_log.info("API: http://{}".format(my_peer.to_string()))
app = NodeApplication(config, mongo, peers)
app_log.info("Starting server on {}:{}".format(config.serve_host,
config.serve_port))
app.listen(config.serve_port, config.serve_host)
if config.ssl:
ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH,
cafile=config.ssl.get('cafile'))
ssl_ctx.load_cert_chain(config.ssl.get('certfile'),
keyfile=config.ssl.get('keyfile'))
http_server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_ctx)
http_server.listen(config.ssl['port'])
# The server will simply run until interrupted
# with Ctrl-C, but if you want to shut down more gracefully,
# call shutdown_event.set().
shutdown_event = tornado.locks.Event()
await shutdown_event.wait()
async def import_block(self, block_data: dict, trigger_event=True) -> bool:
"""Block_data contains peer and block keys. Tries to import that block, retrace if necessary
sends True if that block was inserted, False if it fails or if a retrace was needed.
This is the central entry point for inserting a block, that will modify the local chain and trigger the event,
unless we asked not to, because we're in a batch insert context"""
try:
block = Block.from_dict(block_data['block'])
peer = Peer.from_string(block_data['peer'])
if 'extra_blocks' in block_data:
extra_blocks = None
# extra_blocks = [Block.from_dict( x) for x in block_data['extra_blocks']] # Not used later on, just ram and resources usage
else:
extra_blocks = None
self.app_log.debug("Latest block was {} {} {} {}".format(
self.latest_block.hash, block.prev_hash,
self.latest_block.index, (block.index - 1)))
if int(block.index) > CHAIN.CHECK_TIME_FROM and int(
block.time) < int(self.latest_block.time):
self.app_log.warning(
"New block {} can't be at a sooner time than previous one. Rejecting"
.format(block.index))
await self.mongo.async_db.consensus.update_one(
{
'peer': peer.to_string(),
'index': block.index,
'id': block.signature
}, {'$set': {
'ignore': True
}})
return False
if int(block.index) > CHAIN.CHECK_TIME_FROM and (
int(block.time) <
(int(self.latest_block.time) + 600)) and block.special_min:
self.app_log.warning(
"New special min block {} too soon. Rejecting".format(
block.index))
await self.mongo.async_db.consensus.update_one(
{
'peer': peer.to_string(),
'index': block.index,
'id': block.signature
}, {'$set': {
'ignore': True
}})
return False
try:
result = await self.integrate_block_with_existing_chain(
block, extra_blocks)
if result is False:
# TODO: factorize
await self.mongo.async_db.consensus.update_one(
{
'peer': peer.to_string(),
'index': block.index,
'id': block.signature
}, {'$set': {
'ignore': True
}})
elif trigger_event:
await self.trigger_update_event(block_data['block'])
return result
except DuplicateKeyError as e:
await self.mongo.async_db.consensus.update_one(
{
'peer': peer.to_string(),
'index': block.index,
'id': block.signature
}, {'$set': {
'ignore': True
}})
except AboveTargetException as e:
await self.mongo.async_db.consensus.update_one(
{
'peer': peer.to_string(),
'index': block.index,
'id': block.signature
}, {'$set': {
'ignore': True
}})
except ForkException as e:
await self.retrace(block, peer)
if trigger_event:
await self.trigger_update_event()
return False
except IndexError as e:
await self.retrace(block, peer)
if trigger_event:
await self.trigger_update_event()
return False
except Exception as e:
print("348", e)
exc_type, exc_obj, exc_tb = exc_info()
fname = path.split(exc_tb.tb_frame.f_code.co_filename)[1]
self.app_log.warning("{} {} {}".format(exc_type, fname,
exc_tb.tb_lineno))
await self.mongo.async_db.consensus.update_one(
{
'peer': peer.to_string(),
'index': block.index,
'id': block.signature
}, {'$set': {
'ignore': True
}})
except Exception as e:
exc_type, exc_obj, exc_tb = exc_info()
fname = path.split(exc_tb.tb_frame.f_code.co_filename)[1]
self.app_log.warning("{} {} {}".format(exc_type, fname,
exc_tb.tb_lineno))
if trigger_event:
await self.trigger_update_event()
return False
if trigger_event:
await self.trigger_update_event()
return True
async def search_network_for_new(self):
# Peers.init( self.config.network)
if self.config.network == 'regnet':
return
if self.peers.syncing:
self.app_log.debug(
"Already syncing, ignoring search_network_for_new")
if len(self.config.force_polling):
# This is a temp hack until everyone updated
polling_peers = [
"{}:{}".format(peer['host'], peer['port'])
for peer in self.config.force_polling
]
else:
if len(self.peers.peers) < 2:
await self.peers.refresh()
await async_sleep(20)
if len(self.peers.peers) < 1:
self.app_log.info("No peer to connect to yet")
await async_sleep(10)
return
polling_peers = [peer.to_string() for peer in self.peers.peers]
# TODO: use an aio lock
self.app_log.debug('requesting {} ...'.format(self.latest_block.index +
1))
http_client = AsyncHTTPClient()
# for peer in self.peers.peers:
for peer_string in polling_peers:
self.peers.syncing = True
try:
self.app_log.debug('requesting {} from {}'.format(
self.latest_block.index + 1, peer_string))
peer = Peer.from_string(peer_string)
try:
url = 'http://{peer}/get-blocks?start_index={start_index}&end_index={end_index}'\
.format(peer=peer_string,
start_index=int(self.latest_block.index) +1,
end_index=int(self.latest_block.index) + 100)
request = HTTPRequest(url,
connect_timeout=3,
request_timeout=5)
response = await http_client.fetch(request)
if response.code != 200:
continue
# result = requests.get(url, timeout=2)
except HTTPError as e:
self.app_log.warning(
'Error requesting from {} ...'.format(peer_string))
# add to failed peers
await self.peers.increment_failed(peer)
continue
except ConnectTimeoutError as e:
self.app_log.warning(
'Timeout requesting from {} ...'.format(peer_string))
# add to failed peers
await self.peers.increment_failed(peer)
continue
except Exception as e:
self.app_log.error(
'error {} requesting from {} ...'.format(
e, peer_string))
await self.peers.increment_failed(peer)
continue
try:
blocks = json.loads(response.body.decode('utf-8'))
# blocks = json.loads(result.content)
except ValueError:
continue
inserted = False
for block in blocks:
# print("looking for ", self.existing_blockchain.blocks[-1].index + 1)
block = Block.from_dict(block)
if block.index == (
self.existing_blockchain.blocks[-1].index + 1):
await self.insert_consensus_block(block, peer)
# print("consensus ok", block.index)
res = await self.import_block(
{
'peer': peer_string,
'block': block.to_dict(),
'extra_blocks': blocks
},
trigger_event=False)
# print("import ", block.index, res)
if res:
self.latest_block = block
inserted = True
else:
# 2 cases: bad block, or retrace.
if self.existing_blockchain.blocks[
-1].index == self.latest_block.index:
# bad block, nothing moved, early exit
self.app_log.debug('Bad block {}'.format(
block.index))
else:
# retraced, sync
self.latest_block = Block.from_dict(
await
self.config.BU.get_latest_block_async())
self.app_log.debug('retraced up to {}'.format(
self.latest_block.index))
inserted = True
# in both case, no need to process further blocks
break
else:
break
#print("pass", block.index)
if inserted:
await self.trigger_update_event()
# await self.peers.on_block_insert(self.latest_block.to_dict())
except Exception as e:
if self.debug:
self.app_log.warning(e)
finally:
self.peers.syncing = False
if not Peers.peers:
time.sleep(1)
continue
Faucet.run(config, mongo)
time.sleep(1)
"""
elif args.mode == 'pool':
print("Not supported Yet")
sys.exit()
"""
pp = PoolPayer(config, mongo)
while 1:
pp.do_payout()
time.sleep(1)
"""
elif args.mode == 'serve':
from yadacoin.peers import Peer
from yadacoin.serve import Serve
print(config.to_json())
config.network = args.network
my_peer = Peer.init_my_peer(config, mongo, config.network)
print("http://{}".format(my_peer.to_string()))
app = Flask(__name__)
serve = Serve(config, mongo, app)
app.run(config.serve_host, config.serve_port)