def run(self, chains=None): """Run the main event loop Args: chains (set(str)): Set of chains to operate on. Defaults to {'home', 'side'} """ if chains is None: chains = {'home', 'side'} configure_event_loop() while True: try: asyncio.get_event_loop().run_until_complete( self.run_task(chains=chains)) except asyncio.CancelledError: logger.info('Clean exit requested, exiting') asyncio_join() exit(0) except Exception: logger.exception('Unhandled exception at top level') asyncio_stop() asyncio_join() self.tries += 1 wait = min(MAX_WAIT, self.tries * self.tries) logger.critical( 'Detected unhandled exception, sleeping for %s seconds then resetting task', wait) time.sleep(wait) continue
async def setup(self, loop: asyncio.AbstractEventLoop): self.setup_synchronization(loop) self.setup_graceful_shutdown(loop) await self.setup_liveness(loop) await self.setup_redis(loop) if not await self.scanner.setup(): logger.critical( 'Scanner instance reported unsuccessful setup. Exiting.') exit(1)
async def __get_transactions(self, txhashes, tries, chain, api_key): """Get generated events or errors from receipts for a set of txhashes Args: txhashes (List[str]): The txhashes of the receipts to process tries (int): Number of times to retry before giving upyy chain (str): Which chain to operate on api_key (str): Override default API key Returns: (bool, bool, dict, List[str]): Success, Resync nonce, Response JSON parsed from polyswarmd containing emitted events, errors """ nonce_manager = self.client.nonce_managers[chain] success = False resync_nonce = False results = {} errors = [] while tries > 0: tries -= 1 success, results = await self.client.make_request('GET', '/transactions', chain, json={'transactions': txhashes}, api_key=api_key, tries=1) if not success: if self.client.tx_error_fatal: logger.critical('Received fatal transaction error during get.', extra={'extra': results}) logger.critical(LOG_MSG_ENGINE_TOO_SLOW) exit(1) else: logger.error('Received transaction error during get', extra={'extra': results}) results = {} if results is None else results errors = results.get('errors', []) success = self.has_required_event(results) # Indicates nonce may be too high, if so resync nonces and try again at top level if any(['timeout during wait for receipt' in e.lower() for e in errors]): logger.error('Nonce desync detected during get, resyncing and trying again') nonce_manager.mark_update_nonce() resync_nonce = True break # Check to see if we failed to retrieve some receipts, retry the fetch if so if not success and any(['receipt' in e.lower() for e in errors]): logger.warning('Error fetching some receipts, retrying') continue if any(['transaction failed' in e.lower() for e in errors]): logger.error('Transaction failed due to bad parameters, not retrying', extra={'extra': errors}) break return success, resync_nonce, results, errors
def run(self): while not self.finished: configure_event_loop() loop = asyncio.get_event_loop() try: self.start(loop) # This stops any leftover tasks after out main task finishes asyncio_stop() except asyncio.CancelledError: logger.info('Clean exit requested, exiting') asyncio_join() exit(0) except Exception: logger.exception('Unhandled exception at top level') asyncio_stop() asyncio_join() self.tries += 1 wait = min(MAX_WAIT, self.tries * self.tries) logger.critical( f'Detected unhandled exception, sleeping for {wait} seconds then resetting task' ) time.sleep(wait) continue
def run(self): while not self.finished: loop = asyncio.SelectorEventLoop() # Default event loop does not support pipes on Windows if sys.platform == 'win32': loop = asyncio.ProactorEventLoop() asyncio.set_event_loop(loop) # K8 uses SIGTERM on linux and SIGINT and windows exit_signal = signal.SIGINT if platform.system() == "Windows" else signal.SIGTERM try: loop.add_signal_handler(exit_signal, self.handle_signal) except NotImplementedError: # Disable graceful exit, but run anyway logger.exception(f'{platform.system()} does not support graceful shutdown') try: asyncio.get_event_loop().run_until_complete(self.setup()) gather_task = asyncio.gather(*[self.run_task(i) for i in range(self.task_count)]) asyncio.get_event_loop().run_until_complete(gather_task) except asyncio.CancelledError: logger.info('Clean exit requested, exiting') asyncio_join() exit(0) except Exception: logger.exception('Unhandled exception at top level') asyncio_stop() asyncio_join() self.tries += 1 wait = min(MAX_WAIT, self.tries * self.tries) logger.critical(f'Detected unhandled exception, sleeping for {wait} seconds then resetting task') time.sleep(wait) continue
async def __get_transactions(self, txhashes, nonces, tries, chain, api_key): """Get generated events or errors from receipts for a set of txhashes Args: txhashes (List[str]): The txhashes of the receipts to process tries (int): Number of times to retry before giving upyy chain (str): Which chain to operate on api_key (str): Override default API key Returns: (bool, dict, List[str]): Success, Resync nonce, Response JSON parsed from polyswarmd containing emitted events, errors """ nonce_manager = self.client.nonce_managers[chain] success = False timeout = False results = {} errors = [] while tries > 0: tries -= 1 success, results = await self.client.make_request( 'GET', '/transactions', chain, json={'transactions': txhashes}, api_key=api_key, tries=1) results = {} if results is None else results success = self.has_required_event(results) if not success: if self.client.tx_error_fatal: logger.critical( 'Received fatal transaction error during get.', extra={'extra': results}) exit(1) else: logger.error('Received transaction error during get', extra={'extra': results}) errors = results.get('errors', []) # Indicates nonce may be too high # First, tries to sleep and see if the transaction did succeed (settles can timeout) # If still timing out, resync nonce if any([ e for e in errors if 'timeout during wait for receipt' in e.lower() ]): # We got the items we wanted, but I want it to get all items if success: if not timeout: timeout = True continue # Oh well, it worked so just keep going elif not timeout: logger.error( 'Nonce desync detected during get, resyncing and trying again' ) await nonce_manager.mark_overset_nonce(nonces) # Just give one extra try since settles sometimes timeout tries += 1 timeout = True continue else: logger.error( 'Transaction continues to timeout, sleeping then trying again' ) await asyncio.sleep(1) continue # Check to see if we failed to retrieve some receipts, retry the fetch if so if not success and any(['receipt' in e.lower() for e in errors]): logger.warning('Error fetching some receipts, retrying') continue if any(['transaction failed' in e.lower() for e in errors]): logger.error( 'Transaction failed due to bad parameters, not retrying', extra={'extra': errors}) break # I think we need a break here. We have been just retrying every transaction, even on success break if timeout: logger.warning('Transaction completed after timeout', extra={'extra': results}) return success, results, errors
async def __sign_and_post_transactions(self, transactions, tries, chain, api_key): """Signs and posts a set of transactions to Ethereum via polyswarmd Args: transactions (List[Transaction]): The transactions to sign and post tries (int): Number of times to retry before giving upyy chain (str): Which chain to operate on api_key (str): Override default API key Returns: Response JSON parsed from polyswarmd containing transaction status """ nonce_manager = self.client.nonce_managers[chain] nonces = [] txhashes = [] errors = [] while tries > 0: tries -= 1 while True: nonces = await nonce_manager.reserve(amount=len(transactions)) if nonces is not None: break await asyncio.sleep(1) for i, transaction in enumerate(transactions): transaction['nonce'] = nonces[i] signed_txs = self.client.sign_transactions(transactions) raw_signed_txs = [ bytes(tx['rawTransaction']).hex() for tx in signed_txs if tx.get('rawTransaction', None) is not None ] success, results = await self.client.make_request( 'POST', '/transactions', chain, json={'transactions': raw_signed_txs}, api_key=api_key, tries=1) if not success: # Known transaction errors seem to be a geth issue, don't spam log about it all_known_tx_errors = results is not None and \ all(['known transaction' in r.get('message', '') for r in results if r.get('is_error')]) if self.client.tx_error_fatal: logger.critical( 'Received fatal transaction error during post.', extra={'extra': results}) logger.critical(LOG_MSG_ENGINE_TOO_SLOW) exit(1) elif not all_known_tx_errors: logger.error('Received transaction error during post', extra={ 'extra': { 'results': results, 'transactions': transactions } }) results = [] if results is None else results if len(results) != len(signed_txs): logger.warning('Transaction result length mismatch') txhashes = [] errors = [] for tx, result in zip(signed_txs, results): if tx.get('hash', None) is None: logger.warning(f'Signed transaction missing txhash: {tx}') continue txhash = bytes(tx['hash']).hex() message = result.get('message', '') is_error = result.get('is_error', False) # Known transaction errors seem to be a geth issue, don't retransmit in this case if is_error and 'known transaction' not in message.lower(): errors.append(message) else: txhashes.append(txhash) if txhashes: if errors: logger.warning( 'Transaction errors detected but some succeeded, fetching events', extra={'extra': errors}) return txhashes, nonces, errors # Indicates nonce is too low, we can handle this now, resync nonces and retry if any(['invalid transaction error' in e.lower() for e in errors]): logger.error( 'Nonce desync detected during post, resyncing and trying again' ) await nonce_manager.mark_update_nonce() return txhashes, nonces, errors
async def send(self, chain, tries=2, api_key=None): """Make a transaction generating request to polyswarmd, then sign and post the transactions Args: chain (str): Which chain to operate on api_key (str): Override default API key tries (int): Number of times to retry before giving up Returns: (bool, obj): Tuple of boolean representing success, and response JSON parsed from polyswarmd """ if api_key is None: api_key = self.client.api_key # Ensure we try at least once tries = max(tries, 1) # Step 1: Prepare the transaction, this is only done once success, results = await self.client.make_request('POST', self.get_path(), chain, json=self.get_body(), send_nonce=True, api_key=api_key, tries=tries) results = {} if results is None else results if not success or 'transactions' not in results: logger.error('Expected transactions, received', extra={'extra': results}) return False, results transactions = results.get('transactions', []) if not self.verify(transactions): logger.critical( 'Transactions did not match expectations for the given request.', extra={'extra': transactions}) if self.client.tx_error_fatal: logger.critical(LOG_MSG_ENGINE_TOO_SLOW) exit(1) return False, {} # Keep around any extra data from the first request, such as nonce for assertion if 'transactions' in results: del results['transactions'] orig_tries = tries post_errors = [] get_errors = [] while tries > 0: # Step 2: Update nonces, sign then post transactions txhashes, nonces, post_errors = await self.__sign_and_post_transactions( transactions, orig_tries, chain, api_key) if not txhashes: return False, {'errors': post_errors} # Step 3: At least one transaction was submitted successfully, get and verify the events it generated success, results, get_errors = await self.__get_transactions( txhashes, nonces, orig_tries, chain, api_key) return success, results return False, {'errors': post_errors + get_errors}
async def setup(self): self.scan_lock = asyncio.Semaphore(value=self.scan_limit) self.download_lock = asyncio.Semaphore(value=self.download_limit) if not await self.scanner.setup(): logger.critical('Scanner instance reported unsuccessful setup. Exiting.') exit(1)
async def submit_bounty(self, bounty, chain): """Submit a bounty in a new task Args: bounty (QueuedBounty): Bounty to submit chain: Name of the chain to post to """ assertion_reveal_window = await self.client.bounties.parameters[ chain].get('assertion_reveal_window') arbiter_vote_window = await self.client.bounties.parameters[chain].get( 'arbiter_vote_window') bounty_fee = await self.client.bounties.parameters[chain].get( 'bounty_fee') tries = 0 while tries < MAX_TRIES: balance = await self.client.balances.get_nct_balance(chain) # If we don't have the balance, don't submit. Wait and try a few times, then skip if balance < bounty.amount + bounty_fee: # Skip to next bounty, so one ultra high value bounty doesn't DOS ambassador if self.client.tx_error_fatal and tries >= MAX_TRIES: logger.error( f'Failed {tries} attempts to post bounty due to low balance. Exiting' ) exit(1) return else: tries += 1 logger.critical( f'Insufficient balance to post bounty on {chain}. Have {balance} NCT. ' f'Need {bounty.amount + bounty_fee} NCT.', extra={'extra': bounty}) await asyncio.sleep(tries * tries) continue metadata = None if bounty.metadata is not None: ipfs_hash = await self.client.bounties.post_metadata( bounty.metadata, chain) metadata = ipfs_hash if ipfs_hash is not None else None await self.on_before_bounty_posted(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain) bounties = await self.client.bounties.post_bounty( bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, api_key=bounty.api_key, metadata=metadata) if not bounties: await self.on_bounty_post_failed(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, metadata=bounty.metadata) else: async with self.bounties_posted_locks[chain]: bounties_posted = self.bounties_posted.get(chain, 0) logger.info(f'Submitted bounty {bounties_posted}', extra={'extra': bounty}) self.bounties_posted[chain] = bounties_posted + len( bounties) async with self.bounties_pending_locks[chain]: bounties_pending = self.bounties_pending.get(chain, set()) self.bounties_pending[chain] = bounties_pending | { b.get('guid') for b in bounties if 'guid' in b } for b in bounties: guid = b.get('guid') expiration = int(b.get('expiration', 0)) if guid is None or expiration == 0: logger.error( 'Processing invalid bounty, not scheduling settle') continue # Handle any additional steps in derived implementations await self.on_after_bounty_posted(guid, bounty.artifact_type, bounty.amount, bounty.ipfs_uri, expiration, chain, metadata=bounty.metadata) sb = SettleBounty(guid) self.client.schedule( expiration + assertion_reveal_window + arbiter_vote_window, sb, chain) self.bounty_queues[chain].task_done() self.bounty_semaphores[chain].release() return logger.warning( 'Failed %s attempts to post bounty due to low balance. Skipping', tries, extra={'extra': bounty}) await self.on_bounty_post_failed(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, metadata=bounty.metadata)
async def submit_bounty(self, bounty, chain): """Submit a bounty in a new task Args: bounty (QueuedBounty): Bounty to submit chain: Name of the chain to post to """ bounty_fee = await self.client.bounties.parameters[chain].get( 'bounty_fee') try: await self.client.balances.raise_low_balance( bounty.amount + bounty_fee, chain) except LowBalanceError: await self.client.liveness_recorder.remove_waiting_task( bounty.ipfs_uri) await self.on_bounty_post_failed(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, metadata=bounty.metadata) self.bounty_queues[chain].task_done() self.bounty_semaphores[chain].release() if self.client.tx_error_fatal: logger.error( 'Failed to post bounty due to low balance. Exiting') exit(1) assertion_reveal_window = await self.client.bounties.parameters[ chain].get('assertion_reveal_window') arbiter_vote_window = await self.client.bounties.parameters[chain].get( 'arbiter_vote_window') metadata = None if bounty.metadata is not None: metadata = await self.client.bounties.post_metadata( bounty.metadata, chain) await self.on_before_bounty_posted(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain) bounties = await self.client.bounties.post_bounty( bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, api_key=bounty.api_key, metadata=metadata) await self.client.liveness_recorder.remove_waiting_task(bounty.ipfs_uri ) if not bounties: await self.on_bounty_post_failed(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, metadata=bounty.metadata) else: async with self.bounties_posted_locks[chain]: bounties_posted = self.bounties_posted.get(chain, 0) logger.info('Submitted bounty %s', bounties_posted, extra={'extra': bounty}) self.bounties_posted[chain] = bounties_posted + len(bounties) async with self.bounties_pending_locks[chain]: bounties_pending = self.bounties_pending.get(chain, set()) self.bounties_pending[chain] = bounties_pending | { b.get('guid') for b in bounties if 'guid' in b } for b in bounties: guid = b.get('guid') expiration = int(b.get('expiration', 0)) if guid is None or expiration == 0: logger.error( 'Processing invalid bounty, not scheduling settle') continue # Handle any additional steps in derived implementations await self.on_after_bounty_posted(guid, bounty.artifact_type, bounty.amount, bounty.ipfs_uri, expiration, chain, metadata=bounty.metadata) sb = SettleBounty(guid) self.client.schedule( expiration + assertion_reveal_window + arbiter_vote_window, sb, chain) self.bounty_queues[chain].task_done() self.bounty_semaphores[chain].release()