def main(log, log_format, loop_update_threshold, average_bounty_wait_threshold): if 'liveliness' in sys.argv[0]: warnings.simplefilter('module', category=DeprecationWarning) warnings.warn('liveliness is deprecated, use liveness', DeprecationWarning) loglevel = getattr(logging, log.upper(), None) if not isinstance(loglevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) init_logging(['liveness'], log_format, loglevel) liveness_check = LocalLivenessCheck(loop_update_threshold, average_bounty_wait_threshold) if not liveness_check.check(): raise FatalError('Liveness check failed', 1)
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, allow_key_over_http, chains, watchdog, log_format, submission_rate): """ Entrypoint for the ambassador driver """ utils.fast_deprecation() loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) logger_name, ambassador_class = choose_backend(backend) init_logging(['ambassador', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) if insecure_transport: warnings.warn('--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`', DeprecationWarning) ambassador_class.connect(polyswarmd_addr, keyfile, password, api_key=api_key, testing=testing, chains=set(chains), watchdog=watchdog, submission_rate=submission_rate).run()
async def __handle_new_block(self, number, chain): if number <= self.last_block: return self.last_block = number event = self.block_events.get(chain) if event is not None and number % BLOCK_DIVISOR == 0: event.set() if not self.watchdog: return if not self.first_block: self.first_block = number return blocks = number - self.first_block async with self.bounties_posted_locks[chain]: bounties_posted = self.bounties_posted.get(chain, 0) last_bounty_count = self.last_bounty_count.get(chain, 0) if blocks % self.watchdog == 0 and bounties_posted == last_bounty_count: raise FatalError('Bounties not processing') self.last_bounty_count[chain] = bounties_posted
def main(log, client_log, redis_addr, queue, backend, tasks, download_limit, scan_limit, api_key, testing, log_format, scan_time_requirement, daily_rate_limit, hourly_rate_limit, minutely_rate_limit, secondly_rate_limit, allow_key_over_http): """Entrypoint for the worker driver """ utils.fast_deprecation() if tasks != 0: warnings.warn('Use of --tasks or TASKS is deprecated', DeprecationWarning) tasks = 0 if download_limit == 0 or scan_limit == 0 else max(download_limit, scan_limit) loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) logger_name, scanner_class = choose_backend(backend) scanner = scanner_class() init_logging(['worker', 'microengine', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) logger.info('Running worker with %s tasks', tasks if tasks > 0 else 'unlimited') worker = Worker(redis_addr, queue, tasks, download_limit, scan_limit, api_key, testing, scanner, scan_time_requirement, daily_rate_limit, hourly_rate_limit, minutely_rate_limit, secondly_rate_limit, allow_key_over_http) worker.run()
async def __handle_run(self, chain): """Perform setup required once on correct loop Args: chain (str): Chain we are operating on. """ if self.scanner is not None and not await self.scanner.setup(): raise FatalError('Scanner setup failed', 1)
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, allow_key_over_http, chains, log_format, artifact_type, bid_strategy, accept, exclude, filter, confidence): """ Entrypoint for the microengine driver """ utils.fast_deprecation() loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) if insecure_transport: warnings.warn( '--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`', DeprecationWarning) logger_name, microengine_class = choose_backend(backend) bid_logger_name, bid_strategy_class = choose_bid_strategy(bid_strategy) artifact_types = None init_logging(['microengine', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] filter_accept = filter.get('accept', []) filter_reject = filter.get('reject', []) if accept or exclude: warnings.warn( 'Options `--exclude|accept key:value` are deprecated, please switch to' ' `--filter accept|reject key comparison value`', DeprecationWarning) filter_accept.extend(accept) filter_reject.extend(exclude) favor = confidence.get('favor', []) penalize = confidence.get('penalize', []) microengine_class.connect( polyswarmd_addr, keyfile, password, api_key=api_key, artifact_types=artifact_types, bid_strategy=bid_strategy_class(), bounty_filter=BountyFilter(filter_accept, filter_reject), chains=set(chains), confidence_modifier=ConfidenceModifier(favor, penalize), testing=testing).run()
async def send(self, chain, api_key=None): """Make a transaction generating request to polyswarmd, then sign and post the transactions Args: chain (str): Which chain to operate on api_key (str): Override default API key Returns: (bool, obj): Tuple of boolean representing success, and response JSON parsed from polyswarmd """ if api_key is None: api_key = self.client.api_key # Step 1: Prepare the transaction, this is only done once success, results = await self.client.make_request('POST', self.get_path(), chain, json=self.get_body(), send_nonce=True, api_key=api_key) results = {} if results is None else results if not success or 'transactions' not in results: logger.error('Expected transactions, received', extra={'extra': results}) return False, results transactions = results.get('transactions', []) if not self.verify(transactions): logger.critical( 'Transactions did not match expectations for the given request.', extra={'extra': transactions}) if self.client.tx_error_fatal: logger.critical(LOG_MSG_ENGINE_TOO_SLOW) raise FatalError('Transaction error in testing mode', 1) return False, {} # Keep around any extra data from the first request, such as nonce for assertion if 'transactions' in results: del results['transactions'] # Step 2: Update nonces, sign then post transactions txhashes, nonces, post_errors = await self.__sign_and_post_transactions( transactions, chain, api_key) if not txhashes: return False, {'errors': post_errors} # Step 3: At least one transaction was submitted successfully, get and verify the events it generated success, results, get_errors = await self.__get_transactions( txhashes, nonces, chain, api_key) if not success: return False, {'errors': post_errors + get_errors} else: return True, results
async def __get_transactions(self, txhashes, nonces, chain, api_key): """Get generated events or errors from receipts for a set of txhashes Args: txhashes (List[str]): The txhashes of the receipts to process chain (str): Which chain to operate on api_key (str): Override default API key Returns: (bool, dict, List[str]): Success, Resync nonce, Response JSON parsed from polyswarmd containing emitted events, errors """ loop = asyncio.get_event_loop() nonce_manager = self.client.nonce_managers[chain] success, results = await self.client.make_request( 'GET', '/transactions', chain, json={'transactions': txhashes}, api_key=api_key) results = {} if results is None else results success = self.has_required_event(results) if not success: if self.client.tx_error_fatal: logger.critical('Received fatal transaction error during get.', extra={'extra': results}) raise FatalError('Transaction error in testing mode', 1) else: logger.error('Received transaction error during get', extra={'extra': results}) errors = results.get('errors', []) # Indicates nonce may be too high # First, tries to sleep and see if the transaction did succeed (settles can timeout) if not success and any([ e for e in errors if 'timeout during wait for receipt' in e.lower() ]): logger.error( 'Nonce desync detected during get, resyncing and trying again') loop.create_task(nonce_manager.mark_overset_nonce(nonces)) await asyncio.sleep(1) raise NonceDesyncError # Check to see if we failed to retrieve some receipts, retry the fetch if so if not success and any(['receipt' in e.lower() for e in errors]): logger.warning('Error fetching some receipts, retrying') raise ReceiptError if any(['transaction failed' in e.lower() for e in errors]): logger.error( 'Transaction failed due to bad parameters, not retrying', extra={'extra': errors}) return success, results, errors
def view_stake(polyswarmd_addr, keyfile, password, api_key, testing, insecure_transport, allow_key_over_http, denomination, chain): polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) client = Client(polyswarmd_addr, keyfile, password, api_key, testing > 0) balance = ViewStake(client, denomination, chain) balance.run_oneshot() if balance.exit_code: raise FatalError('Error viewing stake', balance.exit_code)
def cli(log, client_log, log_format): """ Entrypoint for the balance manager driver """ loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) init_logging(['balancemanager'], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel)
async def __handle_run(self, chain): """ If the Client's current balance is less than the minimum stake then deposit the difference between the two to the given chain. Args: chain (str): Chain we are operating on. """ await self.client.balances.get_nct_balance(chain) min_stake = await self.client.staking.parameters[chain].get( 'minimum_stake') staking_balance = await self.client.staking.get_total_balance(chain) if staking_balance < min_stake: try: await self.deposit_stake(min_stake - staking_balance, chain) except LowBalanceError as e: raise FatalError( f'Failed to stake {min_stake - staking_balance} nct due to low balance', 1) from e if self.scanner is not None and not await self.scanner.setup(): raise FatalError('Scanner setup failed', 1)
async def run_task(self): loop = asyncio.get_event_loop() conn = aiohttp.TCPConnector(limit=100) timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT) try: async with self.scanner: async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: async for job in self.get_jobs(): loop.create_task(self.process_job(job, session)) except ScannerSetupFailedError: logger.critical( 'Scanner instance reported unsuccessful setup. Exiting.') raise FatalError('Scanner setup failed', 1)
def withdraw(polyswarmd_addr, keyfile, password, api_key, testing, insecure_transport, allow_key_over_http, denomination, all, amount): """ Withdraw NCT from a sidechain """ polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) if amount is None and not all: raise click.BadArgumentUsage('Must specify either an amount or --all') client = Client(polyswarmd_addr, keyfile, password, api_key, testing > 0) w = Withdraw(client, denomination, all, amount, testing=testing) w.run_oneshot() if w.exit_code: raise FatalError('Error withdrawing NCT', w.exit_code)
def deposit_stake(polyswarmd_addr, keyfile, password, api_key, testing, insecure_transport, allow_key_over_http, denomination, all, chain, amount): """ Deposit NCT into the ArbiterStaking contract """ polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) if amount is None and not all: raise click.BadArgumentUsage('Must specify either an amount or --all') client = Client(polyswarmd_addr, keyfile, password, api_key, testing > 0) d = DepositStake(client, denomination, all, amount, testing=testing, chain=chain) d.run_oneshot() if d.exit_code: raise FatalError('Error depositing stake', d.exit_code)
def main(log, client_log, polyswarmd_addr, keyfile, password, api_key, backend, testing, insecure_transport, allow_key_over_http, chains, log_format, artifact_type): """ Entrypoint for the arbiter driver """ loglevel = getattr(logging, log.upper(), None) clientlevel = getattr(logging, client_log.upper(), None) if not isinstance(loglevel, int) or not isinstance(clientlevel, int): logging.error('invalid log level') raise FatalError('Invalid log level', 1) logger_name, arbiter_class = choose_backend(backend) init_logging(['arbiter', logger_name], log_format, loglevel) init_logging(['polyswarmclient'], log_format, clientlevel) polyswarmd_addr = utils.finalize_polyswarmd_addr(polyswarmd_addr, api_key, allow_key_over_http, insecure_transport) if insecure_transport: warnings.warn( '--insecure-transport will be removed soon. Please add http:// or https:// to polyswarmd-addr`', DeprecationWarning) artifact_types = None if artifact_type: artifact_types = [ ArtifactType.from_string(artifact) for artifact in artifact_type ] arbiter_class.connect(polyswarmd_addr, keyfile, password, api_key=api_key, testing=testing, chains=set(chains), artifact_types=artifact_types).run()
async def __sign_and_post_transactions(self, transactions, chain, api_key): """Signs and posts a set of transactions to Ethereum via polyswarmd Args: transactions (List[Transaction]): The transactions to sign and post chain (str): Which chain to operate on api_key (str): Override default API key Returns: Response JSON parsed from polyswarmd containing transaction status """ loop = asyncio.get_event_loop() nonce_manager = self.client.nonce_managers[chain] errors = [] while True: nonces = await nonce_manager.reserve(amount=len(transactions)) if nonces is not None: break await asyncio.sleep(1) for i, transaction in enumerate(transactions): transaction['nonce'] = nonces[i] results = [] signed_txs = self.__sign_transactions(transactions) for signed_tx, transaction in zip(signed_txs, transactions): success, result = await self.__post_transaction( signed_tx, chain, api_key) results.extend(result) if not success: # Known transaction errors seem to be a geth issue, don't spam log about it if any( ['invalid transaction error' in e.lower() for e in errors]): loop.create_task(nonce_manager.mark_update_nonce()) raise NonceDesyncError all_known_tx_errors = results is not None and \ all(['known transaction' in r.get('message', '') for r in result if r.get('is_error')]) if self.client.tx_error_fatal: logger.critical( 'Received fatal transaction error during post.', extra={'extra': results}) logger.critical(LOG_MSG_ENGINE_TOO_SLOW) raise FatalError('Transaction error in testing mode', 1) elif not all_known_tx_errors: logger.error('Received transaction error during post', extra={ 'extra': { 'results': result, 'transaction': transaction } }) txhashes = [] errors = [] if len(signed_txs) != len(results): raise TransactionError( 'Mistmatch in results counts and transactions') for tx, result in zip(signed_txs, results): if tx.get('hash', None) is None: logger.warning(f'Signed transaction missing txhash: {tx}') continue txhash = bytes(tx['hash']).hex() message = result.get('message', '') is_error = result.get('is_error', False) # Known transaction errors seem to be a geth issue, don't retransmit in this case if is_error and 'known transaction' not in message.lower(): errors.append(message) else: txhashes.append(txhash) if txhashes and errors: logger.warning( 'Transaction errors detected but some succeeded, fetching events', extra={'extra': errors}) return txhashes, nonces, errors
async def submit_bounty(self, bounty, chain): """Submit a bounty in a new task Args: bounty (QueuedBounty): Bounty to submit chain: Name of the chain to post to """ async with self.client.liveness_recorder.waiting_task( bounty.ipfs_uri, self.last_block): bounty_fee = await self.client.bounties.parameters[chain].get( 'bounty_fee') try: await self.client.balances.raise_for_low_balance( bounty.amount + bounty_fee, chain) except LowBalanceError as e: await self.on_bounty_post_failed(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, metadata=bounty.metadata) self.bounty_queues[chain].task_done() self.bounty_semaphores[chain].release() if self.client.tx_error_fatal: raise FatalError( 'Failed to post bounty due to low balance') from e else: return assertion_reveal_window = await self.client.bounties.parameters[ chain].get('assertion_reveal_window') arbiter_vote_window = await self.client.bounties.parameters[ chain].get('arbiter_vote_window') metadata = None if bounty.metadata is not None: metadata = await self.client.bounties.post_metadata( bounty.metadata, chain) await self.on_before_bounty_posted(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain) bounties = await self.client.bounties.post_bounty( bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, api_key=bounty.api_key, metadata=metadata) if not bounties: await self.on_bounty_post_failed(bounty.artifact_type, bounty.amount, bounty.ipfs_uri, bounty.duration, chain, metadata=bounty.metadata) else: async with self.bounties_posted_locks[chain]: bounties_posted = self.bounties_posted.get(chain, 0) logger.info('Submitted bounty %s', bounties_posted, extra={'extra': bounty}) self.bounties_posted[chain] = bounties_posted + len(bounties) async with self.bounties_pending_locks[chain]: bounties_pending = self.bounties_pending.get(chain, set()) self.bounties_pending[chain] = bounties_pending | { b.get('guid') for b in bounties if 'guid' in b } for b in bounties: guid = b.get('guid') expiration = int(b.get('expiration', 0)) if guid is None or expiration == 0: logger.error( 'Processing invalid bounty, not scheduling settle') continue # Handle any additional steps in derived implementations await self.on_after_bounty_posted(guid, bounty.artifact_type, bounty.amount, bounty.ipfs_uri, expiration, chain, metadata=bounty.metadata) sb = SettleBounty(guid) self.client.schedule( expiration + assertion_reveal_window + arbiter_vote_window, sb, chain) self.bounty_queues[chain].task_done() self.bounty_semaphores[chain].release()
async def __handle_new_bounty(self, guid, artifact_type, author, amount, uri, expiration, metadata, block_number, txhash, chain): """Scan and assert on a posted bounty Args: guid (str): The bounty to assert on artifact_type (ArtifactType): The type of artifacts in this bounty author (str): The bounty author amount (str): Amount of the bounty in base NCT units (10 ^ -18) uri (str): IPFS hash of the root artifact expiration (str): Block number of the bounty's expiration metadata (dict): Dictionary of metadata or None block_number (int): Block number the bounty was placed on txhash (str): Transaction hash which caused the event chain (str): Is this on the home or side chain? Returns: Response JSON parsed from polyswarmd containing placed assertions """ # Skip bounties for types we don't support if artifact_type not in self.valid_artifact_types: logger.info('Bounty artifact type %s is not supported', artifact_type) return [] self.bounties_seen += 1 if self.testing > 0: if self.bounties_seen > self.testing: logger.warning( 'Received new bounty, but finished with testing mode') return [] logger.info('Testing mode, %s bounties remaining', self.testing - self.bounties_seen) expiration = int(expiration) duration = expiration - block_number async with self.client.liveness_recorder.waiting_task( guid, block_number): results = await self.fetch_and_scan_all(guid, artifact_type, uri, duration, metadata, chain) mask = [r.bit for r in results] verdicts = [r.verdict for r in results] confidences = [r.confidence for r in results] metadatas = [r.metadata for r in results] combined_metadata = ';'.join(metadatas) try: if all([ metadata and verdict.Verdict.validate(json.loads(metadata)) for metadata in metadatas ]): combined_metadata = json.dumps( [json.loads(metadata) for metadata in metadatas]) except json.JSONDecodeError: logger.exception('Error decoding assertion metadata %s', metadatas) if not any(mask): return [] assertion_fee = await self.client.bounties.parameters[chain].get( 'assertion_fee') assertion_reveal_window = await self.client.bounties.parameters[ chain].get('assertion_reveal_window') arbiter_vote_window = await self.client.bounties.parameters[ chain].get('arbiter_vote_window') bid = await self.bid(guid, mask, verdicts, confidences, metadatas, chain) try: await self.client.balances.raise_for_low_balance( assertion_fee + sum(bid), chain) except LowBalanceError as e: if self.client.tx_error_fatal: raise FatalError( 'Failed to assert on bounty due to low balance') from e return [] logger.info('Responding to %s bounty %s', artifact_type.name.lower(), guid) try: nonce, assertions = await self.client.bounties.post_assertion( guid, bid, mask, verdicts, chain, metadata=combined_metadata) except InvalidMetadataError: logger.exception('Received invalid metadata') return [] for a in assertions: ra = RevealAssertion(guid, a['index'], nonce, verdicts, combined_metadata) self.client.schedule(expiration, ra, chain) sb = SettleBounty(guid) self.client.schedule( expiration + assertion_reveal_window + arbiter_vote_window, sb, chain) return assertions