def run(self, chains=None):
"""Run the main event loop
Args:
chains (set(str)): Set of chains to operate on. Defaults to {'home', 'side'}
"""
if chains is None:
chains = {'home', 'side'}
configure_event_loop()
while True:
try:
asyncio.get_event_loop().run_until_complete(
self.run_task(chains=chains))
except asyncio.CancelledError:
logger.info('Clean exit requested, exiting')
asyncio_join()
exit(0)
except Exception:
logger.exception('Unhandled exception at top level')
asyncio_stop()
asyncio_join()
self.tries += 1
wait = min(MAX_WAIT, self.tries * self.tries)
logger.critical(
'Detected unhandled exception, sleeping for %s seconds then resetting task',
wait)
time.sleep(wait)
continue
async def setup(self, loop: asyncio.AbstractEventLoop):
self.setup_synchronization(loop)
self.setup_graceful_shutdown(loop)
await self.setup_liveness(loop)
await self.setup_redis(loop)
if not await self.scanner.setup():
logger.critical(
'Scanner instance reported unsuccessful setup. Exiting.')
exit(1)
async def __get_transactions(self, txhashes, tries, chain, api_key):
"""Get generated events or errors from receipts for a set of txhashes
Args:
txhashes (List[str]): The txhashes of the receipts to process
tries (int): Number of times to retry before giving upyy
chain (str): Which chain to operate on
api_key (str): Override default API key
Returns:
(bool, bool, dict, List[str]): Success, Resync nonce, Response JSON parsed from polyswarmd containing
emitted events, errors
"""
nonce_manager = self.client.nonce_managers[chain]
success = False
resync_nonce = False
results = {}
errors = []
while tries > 0:
tries -= 1
success, results = await self.client.make_request('GET', '/transactions', chain,
json={'transactions': txhashes}, api_key=api_key, tries=1)
if not success:
if self.client.tx_error_fatal:
logger.critical('Received fatal transaction error during get.', extra={'extra': results})
logger.critical(LOG_MSG_ENGINE_TOO_SLOW)
exit(1)
else:
logger.error('Received transaction error during get', extra={'extra': results})
results = {} if results is None else results
errors = results.get('errors', [])
success = self.has_required_event(results)
# Indicates nonce may be too high, if so resync nonces and try again at top level
if any(['timeout during wait for receipt' in e.lower() for e in errors]):
logger.error('Nonce desync detected during get, resyncing and trying again')
nonce_manager.mark_update_nonce()
resync_nonce = True
break
# Check to see if we failed to retrieve some receipts, retry the fetch if so
if not success and any(['receipt' in e.lower() for e in errors]):
logger.warning('Error fetching some receipts, retrying')
continue
if any(['transaction failed' in e.lower() for e in errors]):
logger.error('Transaction failed due to bad parameters, not retrying', extra={'extra': errors})
break
return success, resync_nonce, results, errors
def run(self):
while not self.finished:
configure_event_loop()
loop = asyncio.get_event_loop()
try:
self.start(loop)
# This stops any leftover tasks after out main task finishes
asyncio_stop()
except asyncio.CancelledError:
logger.info('Clean exit requested, exiting')
asyncio_join()
exit(0)
except Exception:
logger.exception('Unhandled exception at top level')
asyncio_stop()
asyncio_join()
self.tries += 1
wait = min(MAX_WAIT, self.tries * self.tries)
logger.critical(
f'Detected unhandled exception, sleeping for {wait} seconds then resetting task'
)
time.sleep(wait)
continue
def run(self):
while not self.finished:
loop = asyncio.SelectorEventLoop()
# Default event loop does not support pipes on Windows
if sys.platform == 'win32':
loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(loop)
# K8 uses SIGTERM on linux and SIGINT and windows
exit_signal = signal.SIGINT if platform.system() == "Windows" else signal.SIGTERM
try:
loop.add_signal_handler(exit_signal, self.handle_signal)
except NotImplementedError:
# Disable graceful exit, but run anyway
logger.exception(f'{platform.system()} does not support graceful shutdown')
try:
asyncio.get_event_loop().run_until_complete(self.setup())
gather_task = asyncio.gather(*[self.run_task(i) for i in range(self.task_count)])
asyncio.get_event_loop().run_until_complete(gather_task)
except asyncio.CancelledError:
logger.info('Clean exit requested, exiting')
asyncio_join()
exit(0)
except Exception:
logger.exception('Unhandled exception at top level')
asyncio_stop()
asyncio_join()
self.tries += 1
wait = min(MAX_WAIT, self.tries * self.tries)
logger.critical(f'Detected unhandled exception, sleeping for {wait} seconds then resetting task')
time.sleep(wait)
continue
async def __get_transactions(self, txhashes, nonces, tries, chain,
api_key):
"""Get generated events or errors from receipts for a set of txhashes
Args:
txhashes (List[str]): The txhashes of the receipts to process
tries (int): Number of times to retry before giving upyy
chain (str): Which chain to operate on
api_key (str): Override default API key
Returns:
(bool, dict, List[str]): Success, Resync nonce, Response JSON parsed from polyswarmd containing
emitted events, errors
"""
nonce_manager = self.client.nonce_managers[chain]
success = False
timeout = False
results = {}
errors = []
while tries > 0:
tries -= 1
success, results = await self.client.make_request(
'GET',
'/transactions',
chain,
json={'transactions': txhashes},
api_key=api_key,
tries=1)
results = {} if results is None else results
success = self.has_required_event(results)
if not success:
if self.client.tx_error_fatal:
logger.critical(
'Received fatal transaction error during get.',
extra={'extra': results})
exit(1)
else:
logger.error('Received transaction error during get',
extra={'extra': results})
errors = results.get('errors', [])
# Indicates nonce may be too high
# First, tries to sleep and see if the transaction did succeed (settles can timeout)
# If still timing out, resync nonce
if any([
e for e in errors
if 'timeout during wait for receipt' in e.lower()
]):
# We got the items we wanted, but I want it to get all items
if success:
if not timeout:
timeout = True
continue
# Oh well, it worked so just keep going
elif not timeout:
logger.error(
'Nonce desync detected during get, resyncing and trying again'
)
await nonce_manager.mark_overset_nonce(nonces)
# Just give one extra try since settles sometimes timeout
tries += 1
timeout = True
continue
else:
logger.error(
'Transaction continues to timeout, sleeping then trying again'
)
await asyncio.sleep(1)
continue
# Check to see if we failed to retrieve some receipts, retry the fetch if so
if not success and any(['receipt' in e.lower() for e in errors]):
logger.warning('Error fetching some receipts, retrying')
continue
if any(['transaction failed' in e.lower() for e in errors]):
logger.error(
'Transaction failed due to bad parameters, not retrying',
extra={'extra': errors})
break
# I think we need a break here. We have been just retrying every transaction, even on success
break
if timeout:
logger.warning('Transaction completed after timeout',
extra={'extra': results})
return success, results, errors
async def __sign_and_post_transactions(self, transactions, tries, chain,
api_key):
"""Signs and posts a set of transactions to Ethereum via polyswarmd
Args:
transactions (List[Transaction]): The transactions to sign and post
tries (int): Number of times to retry before giving upyy
chain (str): Which chain to operate on
api_key (str): Override default API key
Returns:
Response JSON parsed from polyswarmd containing transaction status
"""
nonce_manager = self.client.nonce_managers[chain]
nonces = []
txhashes = []
errors = []
while tries > 0:
tries -= 1
while True:
nonces = await nonce_manager.reserve(amount=len(transactions))
if nonces is not None:
break
await asyncio.sleep(1)
for i, transaction in enumerate(transactions):
transaction['nonce'] = nonces[i]
signed_txs = self.client.sign_transactions(transactions)
raw_signed_txs = [
bytes(tx['rawTransaction']).hex() for tx in signed_txs
if tx.get('rawTransaction', None) is not None
]
success, results = await self.client.make_request(
'POST',
'/transactions',
chain,
json={'transactions': raw_signed_txs},
api_key=api_key,
tries=1)
if not success:
# Known transaction errors seem to be a geth issue, don't spam log about it
all_known_tx_errors = results is not None and \
all(['known transaction' in r.get('message', '') for r in results if
r.get('is_error')])
if self.client.tx_error_fatal:
logger.critical(
'Received fatal transaction error during post.',
extra={'extra': results})
logger.critical(LOG_MSG_ENGINE_TOO_SLOW)
exit(1)
elif not all_known_tx_errors:
logger.error('Received transaction error during post',
extra={
'extra': {
'results': results,
'transactions': transactions
}
})
results = [] if results is None else results
if len(results) != len(signed_txs):
logger.warning('Transaction result length mismatch')
txhashes = []
errors = []
for tx, result in zip(signed_txs, results):
if tx.get('hash', None) is None:
logger.warning(f'Signed transaction missing txhash: {tx}')
continue
txhash = bytes(tx['hash']).hex()
message = result.get('message', '')
is_error = result.get('is_error', False)
# Known transaction errors seem to be a geth issue, don't retransmit in this case
if is_error and 'known transaction' not in message.lower():
errors.append(message)
else:
txhashes.append(txhash)
if txhashes:
if errors:
logger.warning(
'Transaction errors detected but some succeeded, fetching events',
extra={'extra': errors})
return txhashes, nonces, errors
# Indicates nonce is too low, we can handle this now, resync nonces and retry
if any(['invalid transaction error' in e.lower() for e in errors]):
logger.error(
'Nonce desync detected during post, resyncing and trying again'
)
await nonce_manager.mark_update_nonce()
return txhashes, nonces, errors
async def send(self, chain, tries=2, api_key=None):
"""Make a transaction generating request to polyswarmd, then sign and post the transactions
Args:
chain (str): Which chain to operate on
api_key (str): Override default API key
tries (int): Number of times to retry before giving up
Returns:
(bool, obj): Tuple of boolean representing success, and response JSON parsed from polyswarmd
"""
if api_key is None:
api_key = self.client.api_key
# Ensure we try at least once
tries = max(tries, 1)
# Step 1: Prepare the transaction, this is only done once
success, results = await self.client.make_request('POST',
self.get_path(),
chain,
json=self.get_body(),
send_nonce=True,
api_key=api_key,
tries=tries)
results = {} if results is None else results
if not success or 'transactions' not in results:
logger.error('Expected transactions, received',
extra={'extra': results})
return False, results
transactions = results.get('transactions', [])
if not self.verify(transactions):
logger.critical(
'Transactions did not match expectations for the given request.',
extra={'extra': transactions})
if self.client.tx_error_fatal:
logger.critical(LOG_MSG_ENGINE_TOO_SLOW)
exit(1)
return False, {}
# Keep around any extra data from the first request, such as nonce for assertion
if 'transactions' in results:
del results['transactions']
orig_tries = tries
post_errors = []
get_errors = []
while tries > 0:
# Step 2: Update nonces, sign then post transactions
txhashes, nonces, post_errors = await self.__sign_and_post_transactions(
transactions, orig_tries, chain, api_key)
if not txhashes:
return False, {'errors': post_errors}
# Step 3: At least one transaction was submitted successfully, get and verify the events it generated
success, results, get_errors = await self.__get_transactions(
txhashes, nonces, orig_tries, chain, api_key)
return success, results
return False, {'errors': post_errors + get_errors}
async def setup(self):
self.scan_lock = asyncio.Semaphore(value=self.scan_limit)
self.download_lock = asyncio.Semaphore(value=self.download_limit)
if not await self.scanner.setup():
logger.critical('Scanner instance reported unsuccessful setup. Exiting.')
exit(1)
async def submit_bounty(self, bounty, chain):
"""Submit a bounty in a new task
Args:
bounty (QueuedBounty): Bounty to submit
chain: Name of the chain to post to
"""
assertion_reveal_window = await self.client.bounties.parameters[
chain].get('assertion_reveal_window')
arbiter_vote_window = await self.client.bounties.parameters[chain].get(
'arbiter_vote_window')
bounty_fee = await self.client.bounties.parameters[chain].get(
'bounty_fee')
tries = 0
while tries < MAX_TRIES:
balance = await self.client.balances.get_nct_balance(chain)
# If we don't have the balance, don't submit. Wait and try a few times, then skip
if balance < bounty.amount + bounty_fee:
# Skip to next bounty, so one ultra high value bounty doesn't DOS ambassador
if self.client.tx_error_fatal and tries >= MAX_TRIES:
logger.error(
f'Failed {tries} attempts to post bounty due to low balance. Exiting'
)
exit(1)
return
else:
tries += 1
logger.critical(
f'Insufficient balance to post bounty on {chain}. Have {balance} NCT. '
f'Need {bounty.amount + bounty_fee} NCT.',
extra={'extra': bounty})
await asyncio.sleep(tries * tries)
continue
metadata = None
if bounty.metadata is not None:
ipfs_hash = await self.client.bounties.post_metadata(
bounty.metadata, chain)
metadata = ipfs_hash if ipfs_hash is not None else None
await self.on_before_bounty_posted(bounty.artifact_type,
bounty.amount, bounty.ipfs_uri,
bounty.duration, chain)
bounties = await self.client.bounties.post_bounty(
bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
bounty.duration,
chain,
api_key=bounty.api_key,
metadata=metadata)
if not bounties:
await self.on_bounty_post_failed(bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
bounty.duration,
chain,
metadata=bounty.metadata)
else:
async with self.bounties_posted_locks[chain]:
bounties_posted = self.bounties_posted.get(chain, 0)
logger.info(f'Submitted bounty {bounties_posted}',
extra={'extra': bounty})
self.bounties_posted[chain] = bounties_posted + len(
bounties)
async with self.bounties_pending_locks[chain]:
bounties_pending = self.bounties_pending.get(chain, set())
self.bounties_pending[chain] = bounties_pending | {
b.get('guid')
for b in bounties if 'guid' in b
}
for b in bounties:
guid = b.get('guid')
expiration = int(b.get('expiration', 0))
if guid is None or expiration == 0:
logger.error(
'Processing invalid bounty, not scheduling settle')
continue
# Handle any additional steps in derived implementations
await self.on_after_bounty_posted(guid,
bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
expiration,
chain,
metadata=bounty.metadata)
sb = SettleBounty(guid)
self.client.schedule(
expiration + assertion_reveal_window + arbiter_vote_window,
sb, chain)
self.bounty_queues[chain].task_done()
self.bounty_semaphores[chain].release()
return
logger.warning(
'Failed %s attempts to post bounty due to low balance. Skipping',
tries,
extra={'extra': bounty})
await self.on_bounty_post_failed(bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
bounty.duration,
chain,
metadata=bounty.metadata)
async def submit_bounty(self, bounty, chain):
"""Submit a bounty in a new task
Args:
bounty (QueuedBounty): Bounty to submit
chain: Name of the chain to post to
"""
bounty_fee = await self.client.bounties.parameters[chain].get(
'bounty_fee')
try:
await self.client.balances.raise_low_balance(
bounty.amount + bounty_fee, chain)
except LowBalanceError:
await self.client.liveness_recorder.remove_waiting_task(
bounty.ipfs_uri)
await self.on_bounty_post_failed(bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
bounty.duration,
chain,
metadata=bounty.metadata)
self.bounty_queues[chain].task_done()
self.bounty_semaphores[chain].release()
if self.client.tx_error_fatal:
logger.error(
'Failed to post bounty due to low balance. Exiting')
exit(1)
assertion_reveal_window = await self.client.bounties.parameters[
chain].get('assertion_reveal_window')
arbiter_vote_window = await self.client.bounties.parameters[chain].get(
'arbiter_vote_window')
metadata = None
if bounty.metadata is not None:
metadata = await self.client.bounties.post_metadata(
bounty.metadata, chain)
await self.on_before_bounty_posted(bounty.artifact_type, bounty.amount,
bounty.ipfs_uri, bounty.duration,
chain)
bounties = await self.client.bounties.post_bounty(
bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
bounty.duration,
chain,
api_key=bounty.api_key,
metadata=metadata)
await self.client.liveness_recorder.remove_waiting_task(bounty.ipfs_uri
)
if not bounties:
await self.on_bounty_post_failed(bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
bounty.duration,
chain,
metadata=bounty.metadata)
else:
async with self.bounties_posted_locks[chain]:
bounties_posted = self.bounties_posted.get(chain, 0)
logger.info('Submitted bounty %s',
bounties_posted,
extra={'extra': bounty})
self.bounties_posted[chain] = bounties_posted + len(bounties)
async with self.bounties_pending_locks[chain]:
bounties_pending = self.bounties_pending.get(chain, set())
self.bounties_pending[chain] = bounties_pending | {
b.get('guid')
for b in bounties if 'guid' in b
}
for b in bounties:
guid = b.get('guid')
expiration = int(b.get('expiration', 0))
if guid is None or expiration == 0:
logger.error(
'Processing invalid bounty, not scheduling settle')
continue
# Handle any additional steps in derived implementations
await self.on_after_bounty_posted(guid,
bounty.artifact_type,
bounty.amount,
bounty.ipfs_uri,
expiration,
chain,
metadata=bounty.metadata)
sb = SettleBounty(guid)
self.client.schedule(
expiration + assertion_reveal_window + arbiter_vote_window, sb,
chain)
self.bounty_queues[chain].task_done()
self.bounty_semaphores[chain].release()