class Client(object): """Client to connected to a Ethereum wallet as well as a polyswarmd instance. Args: polyswarmd_addr (str): URI of polyswarmd you are referring to. keyfile (str): Keyfile filename. password (str): Password associated with keyfile. api_key (str): Your PolySwarm API key. tx_error_fatal (bool): Transaction errors are fatal and exit the program insecure_transport (bool): Allow insecure transport such as HTTP? """ def __init__(self, polyswarmd_addr, keyfile, password, api_key=None, tx_error_fatal=False, insecure_transport=False): if api_key and insecure_transport: raise ValueError( 'Refusing to send API key over insecure transport') protocol = 'http://' if insecure_transport else 'https://' self.polyswarmd_uri = protocol + polyswarmd_addr logger.debug('self.polyswarmd_uri: %s', self.polyswarmd_uri) self.api_key = api_key self.tx_error_fatal = tx_error_fatal self.params = {} with open(keyfile, 'r') as f: self.priv_key = w3.eth.account.decrypt(f.read(), password) self.account = w3.eth.account.privateKeyToAccount( self.priv_key).address logger.info('Using account: %s', self.account) self.__session = None # Do not init nonce manager here. Need to wait until we can guarantee that our event loop is set. self.nonce_managers = {} self.__schedules = {} self.tries = 0 self.bounties = None self.staking = None self.offers = None self.relay = None self.balances = None # Setup a liveness instance self.liveness_recorder = LocalLivenessRecorder() # Events from client self.on_run = events.OnRunCallback() # Events from polyswarmd self.on_new_block = events.OnNewBlockCallback() self.on_new_bounty = events.OnNewBountyCallback() self.on_new_assertion = events.OnNewAssertionCallback() self.on_reveal_assertion = events.OnRevealAssertionCallback() self.on_new_vote = events.OnNewVoteCallback() self.on_quorum_reached = events.OnQuorumReachedCallback() self.on_settled_bounty = events.OnSettledBountyCallback() self.on_initialized_channel = events.OnInitializedChannelCallback() self.on_deprecated = events.OnDeprecatedCallback() # Events scheduled on block deadlines self.on_reveal_assertion_due = events.OnRevealAssertionDueCallback() self.on_vote_on_bounty_due = events.OnVoteOnBountyDueCallback() self.on_settle_bounty_due = events.OnSettleBountyDueCallback() self.on_withdraw_stake_due = events.OnWithdrawStakeDueCallback() def run(self, chains=None): """Run the main event loop Args: chains (set(str)): Set of chains to operate on. Defaults to {'home', 'side'} """ if chains is None: chains = {'home', 'side'} configure_event_loop() while True: try: asyncio.get_event_loop().run_until_complete( self.run_task(chains=chains)) except asyncio.CancelledError: logger.info('Clean exit requested, exiting') asyncio_join() exit(0) except Exception: logger.exception('Unhandled exception at top level') asyncio_stop() asyncio_join() self.tries += 1 wait = min(MAX_WAIT, self.tries * self.tries) logger.critical( 'Detected unhandled exception, sleeping for %s seconds then resetting task', wait) time.sleep(wait) continue async def run_task(self, chains=None, listen_for_events=True): """ How the event loop handles running a task. Args: chains (set(str)): Set of chains to operate on. Defaults to {'home', 'side'} """ if chains is None: chains = {'home', 'side'} if self.api_key and not self.polyswarmd_uri.startswith('https://'): raise Exception('Refusing to send API key over insecure transport') self.params = {'account': self.account} # We can now create our locks, because we are assured that the event loop is set self.nonce_managers = { chain: NonceManager(self, chain) for chain in chains } self.__schedules = {chain: events.Schedule() for chain in chains} await self.liveness_recorder.start() try: # XXX: Set the timeouts here to reasonable values, probably should be configurable # No limits on connections conn = aiohttp.TCPConnector(limit=100) timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT) async with aiohttp.ClientSession( connector=conn, timeout=timeout) as self.__session: self.bounties = BountiesClient(self) self.staking = StakingClient(self) self.offers = OffersClient(self) self.relay = RelayClient(self) self.balances = BalanceClient(self) for chain in chains: await self.bounties.fetch_parameters(chain) await self.staking.fetch_parameters(chain) await self.on_run.run(chain) # At this point we're initialized, reset our failure counter and listen for events self.tries = 0 if listen_for_events: await asyncio.wait( [self.listen_for_events(chain) for chain in chains]) finally: self.__session = None self.bounties = None self.staking = None self.offers = None async def make_request(self, method, path, chain, json=None, send_nonce=False, api_key=None, tries=2, params=None): """Make a request to polyswarmd, expecting a json response Args: method (str): HTTP method to use path (str): Path portion of URI to send request to chain (str): Which chain to operate on json (obj): JSON payload to send with request send_nonce (bool): Whether to include a base_nonce query string parameter in this request api_key (str): Override default API key tries (int): Number of times to retry before giving up params (dict): Optional params for the request Returns: (bool, obj): Tuple of boolean representing success, and response JSON parsed from polyswarmd """ if chain != 'home' and chain != 'side': raise ValueError( 'Chain parameter must be `home` or `side`, got {0}'.format( chain)) if self.__session is None or self.__session.closed: raise Exception('Not running') # Ensure we try at least once tries = max(tries, 1) uri = f'{self.polyswarmd_uri}{path}' logger.debug('making request to url: %s', uri) if params is None: params = dict() params.update(dict(self.params)) params['chain'] = chain if send_nonce: # Set to 0 because I will replace it later params['base_nonce'] = 0 # Allow overriding API key per request if api_key is None: api_key = self.api_key headers = {'Authorization': api_key} if api_key is not None else None qs = '&'.join([a + '=' + str(b) for (a, b) in params.items()]) response = {} while tries > 0: tries -= 1 response = {} try: async with self.__session.request(method, uri, params=params, headers=headers, json=json) as raw_response: try: # Handle "Too many requests" rate limit by not hammering server, and instead sleeping a bit if raw_response.status == 429: logger.warning( 'Hit polyswarmd rate limits, sleeping then trying again' ) await asyncio.sleep(RATE_LIMIT_SLEEP) tries += 1 continue response = await raw_response.json() except (ValueError, aiohttp.ContentTypeError): response = await raw_response.read( ) if raw_response else 'None' logger.error( 'Received non-json response from polyswarmd: %s, url: %s', response, uri) response = {} continue except (OSError, aiohttp.ServerDisconnectedError): logger.error('Connection to polyswarmd refused, retrying') except asyncio.TimeoutError: logger.error('Connection to polyswarmd timed out, retrying') logger.debug('%s %s?%s', method, path, qs, extra={'extra': response}) if not check_response(response): if tries > 0: logger.info('Request %s %s?%s failed, retrying...', method, path, qs) continue else: logger.warning('Request %s %s?%s failed, giving up', method, path, qs) return False, response.get('errors') return True, response.get('result') return False, response.get('errors') def sign_transactions(self, transactions): """Sign a set of transactions Args: transactions (List[Transaction]): The transactions to sign Returns: List[Transaction]: The signed transactions """ return [ w3.eth.account.signTransaction(tx, self.priv_key) for tx in transactions ] async def get_base_nonce(self, chain, ignore_pending=False, api_key=None): """Get account's nonce from polyswarmd Args: chain (str): Which chain to operate on ignore_pending (bool): Whether to include pending transactions in nonce or not api_key (str): Override default API key """ params = {'ignore_pending': ' '} if ignore_pending else None success, base_nonce = await self.make_request('GET', '/nonce', chain, api_key=api_key, params=params) if success: return base_nonce else: logger.error('Failed to fetch base nonce') return None async def get_pending_nonces(self, chain, api_key=None): """Get account's pending nonces from polyswarmd Args: chain (str): Which chain to operate on api_key (str): Override default API key """ success, nonces = await self.make_request('GET', '/pending', chain, api_key=api_key) if success: return [int(nonce) for nonce in nonces] else: logger.error('Failed to fetch base nonce') return [] async def list_artifacts(self, ipfs_uri, api_key=None, tries=2): """Return a list of artificats from a given ipfs_uri. Args: ipfs_uri (str): IPFS URI to get artifiacts from. api_key (str): Override default API key Returns: List[(str, str)]: A list of tuples. First tuple element is the artifact name, second tuple element is the artifact hash. """ if not is_valid_ipfs_uri(ipfs_uri): logger.warning('Invalid IPFS URI: %s', ipfs_uri) return [] path = f'/artifacts/{ipfs_uri}' # Chain parameter doesn't matter for artifacts, just set to side success, result = await self.make_request('GET', path, 'side', api_key=api_key, tries=tries) if not success: logger.error('Expected artifact listing, received', extra={'extra': result}) return [] result = {} if result is None else result return [(a.get('name', ''), a.get('hash', '')) for a in result] async def get_artifact_count(self, ipfs_uri, api_key=None): """Gets the number of artifacts at the ipfs uri Args: ipfs_uri (str): IPFS URI for the artifact set api_key (str): Override default API key Returns: Number of artifacts at the uri """ artifacts = await self.list_artifacts(ipfs_uri, api_key=api_key) return len(artifacts) if artifacts is not None and artifacts else 0 async def get_artifact(self, ipfs_uri, index, api_key=None, tries=2): """Retrieve an artifact from IPFS via polyswarmd Args: ipfs_uri (str): IPFS hash of the artifact to retrieve index (int): Index of the sub artifact to retrieve api_key (str): Override default API key Returns: (bytes): Content of the artifact """ if not is_valid_ipfs_uri(ipfs_uri): raise ValueError('Invalid IPFS URI') uri = f'{self.polyswarmd_uri}/artifacts/{ipfs_uri}/{index}' logger.debug('getting artifact from uri: %s', uri) params = dict(self.params) # Allow overriding API key per request if api_key is None: api_key = self.api_key headers = {'Authorization': api_key} if api_key is not None else None while tries > 0: tries -= 1 try: async with self.__session.get(uri, params=params, headers=headers) as raw_response: # Handle "Too many requests" rate limit by not hammering server, and instead sleeping a bit if raw_response.status == 429: logger.warning( 'Hit polyswarmd rate limits, sleeping then trying again' ) await asyncio.sleep(RATE_LIMIT_SLEEP) tries += 1 continue if raw_response.status == 200: return await raw_response.read() except (OSError, aiohttp.ServerDisconnectedError): logger.error('Connection to polyswarmd refused') except asyncio.TimeoutError: logger.error('Connection to polyswarmd timed out') return None @staticmethod def to_wei(amount, unit='ether'): return w3.toWei(amount, unit) @staticmethod def from_wei(amount, unit='ether'): return w3.fromWei(amount, unit) @staticmethod def get_artifact_bid_at_(mask, bid, index): if not mask[index]: return 0 bid_index = sum(mask[:index]) return bid[bid_index] # Async iterator helper class class __GetArtifacts(object): def __init__(self, client, ipfs_uri, api_key=None): self.i = 0 self.client = client self.ipfs_uri = ipfs_uri self.api_key = api_key async def __aiter__(self): return self async def __anext__(self): if not is_valid_ipfs_uri(self.ipfs_uri): raise StopAsyncIteration i = self.i self.i += 1 if i < MAX_ARTIFACTS: content = await self.client.get_artifact(self.ipfs_uri, i, api_key=self.api_key) if content: return content raise StopAsyncIteration def get_artifacts(self, ipfs_uri, api_key=None): """Get an iterator to return artifacts. Args: ipfs_uri (str): URI where artificats are located api_key (str): Override default API key Returns: `__GetArtifacts` iterator """ if self.__session is None or self.__session.closed: raise Exception('Not running') return Client.__GetArtifacts(self, ipfs_uri, api_key=api_key) async def post_artifacts(self, files, api_key=None, tries=2): """Post artifacts to polyswarmd, flexible files parameter to support different use-cases Args: files (list[(filename, contents)]): The artifacts to upload, accepts one of: (filename, bytes): File name and contents to upload (filename, file_obj): (Optional) file name and file object to upload (filename, None): File name to open and upload api_key (str): Override default API key Returns: (str): IPFS URI of the uploaded artifact """ uri = f'{self.polyswarmd_uri}/artifacts' logger.debug('posting artifact to uri: %s', uri) params = dict(self.params) # Allow overriding API key per request if api_key is None: api_key = self.api_key headers = {'Authorization': api_key} if api_key is not None else None while tries > 0: tries -= 1 # MultipartWriter can only be used once, recreate if on retry with aiohttp.MultipartWriter('form-data') as mpwriter: response = {} to_close = [] try: for filename, f in files: # If contents is None, open filename for reading and remember to close it if f is None: f = open(filename, 'rb') to_close.append(f) # If filename is None and our file object has a name attribute, use it if filename is None and hasattr(f, 'name'): filename = f.name if filename: filename = os.path.basename(filename) else: filename = 'file' payload = aiohttp.payload.get_payload( f, content_type='application/octet-stream') payload.set_content_disposition('form-data', name='file', filename=filename) mpwriter.append_payload(payload) # Make the request async with self.__session.post( uri, params=params, headers=headers, data=mpwriter) as raw_response: try: # Handle "Too many requests" rate limit by not hammering server, and instead sleeping a bit if raw_response.status == 429: logger.warning( 'Hit polyswarmd rate limits, sleeping then trying again' ) await asyncio.sleep(RATE_LIMIT_SLEEP) tries += 1 continue response = await raw_response.json() except (ValueError, aiohttp.ContentTypeError): response = await raw_response.read( ) if raw_response else 'None' logger.error( 'Received non-json response from polyswarmd: %s, uri: %s', response, uri) response = {} continue except (OSError, aiohttp.ServerDisconnectedError): logger.error('Connection to polyswarmd refused, files: %s', files) except asyncio.TimeoutError: logger.error( 'Connection to polyswarmd timed out, files: %s', files) finally: for f in to_close: f.close() logger.debug('POST/artifacts', extra={'extra': response}) if not check_response(response): if tries > 0: logger.info( 'Posting artifacts to polyswarmd failed, retrying') continue else: logger.info( 'Posting artifacts to polyswarmd failed, giving up' ) return None return response.get('result') def schedule(self, expiration, event, chain): """Schedule an event to execute on a particular block Args: expiration (int): Which block to execute on event (Event): Event to trigger on expiration block chain (str): Which chain to operate on """ if chain != 'home' and chain != 'side': raise ValueError( 'Chain parameter must be `home` or `side`, got {0}'.format( chain)) self.__schedules[chain].put(expiration, event) async def __handle_scheduled_events(self, number, chain): """Perform scheduled events when a new block is reported Args: number (int): The current block number reported from polyswarmd chain (str): Which chain to operate on """ if chain != 'home' and chain != 'side': raise ValueError( 'Chain parameter must be `home` or `side`, got {0}'.format( chain)) while self.__schedules[chain].peek( ) and self.__schedules[chain].peek()[0] < number: exp, task = self.__schedules[chain].get() if isinstance(task, events.RevealAssertion): asyncio.get_event_loop().create_task( self.on_reveal_assertion_due.run(bounty_guid=task.guid, index=task.index, nonce=task.nonce, verdicts=task.verdicts, metadata=task.metadata, chain=chain)) elif isinstance(task, events.SettleBounty): asyncio.get_event_loop().create_task( self.on_settle_bounty_due.run(bounty_guid=task.guid, chain=chain)) elif isinstance(task, events.VoteOnBounty): asyncio.get_event_loop().create_task( self.on_vote_on_bounty_due.run( bounty_guid=task.guid, votes=task.votes, valid_bloom=task.valid_bloom, chain=chain)) elif isinstance(task, events.WithdrawStake): asyncio.get_event_loop().create_task( self.on_withdraw_stake_due.run(amount=task.amount, chain=chain)) async def listen_for_events(self, chain): """Listen for events via websocket connection to polyswarmd Args: chain (str): Which chain to operate on """ if chain != 'home' and chain != 'side': raise ValueError( 'Chain parameter must be `home` or `side`, got {0}'.format( chain)) if not self.polyswarmd_uri.startswith('http'): raise ValueError( 'polyswarmd_uri protocol is not http or https, got {0}'.format( self.polyswarmd_uri)) # http:// -> ws://, https:// -> wss:// wsuri = f'{self.polyswarmd_uri.replace("http", "ws", 1)}/events?chain={chain}' last_block = 0 retry = 0 while True: try: async with websockets.connect(wsuri) as ws: # Fetch parameters again here so we don't miss update events await self.bounties.fetch_parameters(chain) await self.staking.fetch_parameters(chain) if retry != 0: logger.error( 'Websocket connection to polyswarmd reestablished') retry = 0 while not ws.closed: resp = None try: resp = await ws.recv() resp = json.loads(resp) event = resp.get('event') data = resp.get('data') block_number = resp.get('block_number') txhash = resp.get('txhash') except json.JSONDecodeError: logger.error( 'Invalid event response from polyswarmd: %s', resp) continue except websockets.exceptions.ConnectionClosed: # Trigger retry logic outside main loop break if event != 'block': logger.info('Received %s on chain %s', event, chain, extra={'extra': data}) if event == 'connected': logger.info('Connected to event socket at: %s', data.get('start_time')) elif event == 'block': number = data.get('number', 0) if number <= last_block: continue if number % 100 == 0: logger.debug('Block %s on chain %s', number, chain) asyncio.get_event_loop().create_task( self.on_new_block.run(number=number, chain=chain)) asyncio.get_event_loop().create_task( self.__handle_scheduled_events(number, chain=chain)) asyncio.get_event_loop().create_task( self.liveness_recorder.advance_time(number)) elif event == 'fee_update': d = { 'bounty_fee': data.get('bounty_fee'), 'assertion_fee': data.get('assertion_fee') } await self.bounties.parameters[chain].update( {k: v for k, v in d.items() if v is not None}) elif event == 'window_update': d = { 'assertion_reveal_window': data.get('assertion_reveal_window'), 'arbiter_vote_window': data.get('arbiter_vote_window') } await self.bounties.parameters[chain].update( {k: v for k, v in d.items() if v is not None}) elif event == 'bounty': asyncio.get_event_loop().create_task( self.on_new_bounty.run( **data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'assertion': asyncio.get_event_loop().create_task( self.on_new_assertion.run( **data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'reveal': asyncio.get_event_loop().create_task( self.on_reveal_assertion.run( **data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'vote': asyncio.get_event_loop().create_task( self.on_new_vote.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'quorum': asyncio.get_event_loop().create_task( self.on_quorum_reached.run( **data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'settled_bounty': asyncio.get_event_loop().create_task( self.on_settled_bounty.run( **data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'deprecated': asyncio.get_event_loop().create_task( self.on_deprecated.run( **data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'initialized_channel': asyncio.get_event_loop().create_task( self.on_initialized_channel.run( **data, block_number=block_number, txhash=txhash)) else: logger.error( 'Invalid event type from polyswarmd: %s', resp) except (OSError, websockets.exceptions.InvalidHandshake): logger.error( 'Websocket connection to polyswarmd refused, retrying') except asyncio.TimeoutError: logger.error( 'Websocket connection to polyswarmd timed out, retrying') retry += 1 wait = min(MAX_WAIT, retry * retry) logger.error( 'Websocket connection to polyswarmd closed, sleeping for %s seconds then reconnecting', wait) await asyncio.sleep(wait)
class Client(object): """Client to connected to a Ethereum wallet as well as a polyswarmd instance. Args: polyswarmd_addr (str): URI of polyswarmd you are referring to. keyfile (str): Keyfile filename. password (str): Password associated with keyfile. api_key (str): Your PolySwarm API key. tx_error_fatal (bool): Transaction errors are fatal and exit the program insecure_transport (bool): Allow insecure transport such as HTTP? """ def __init__(self, polyswarmd_addr, keyfile, password, api_key=None, tx_error_fatal=False): self.polyswarmd_uri = polyswarmd_addr logger.debug('self.polyswarmd_uri: %s', self.polyswarmd_uri) self.api_key = api_key self.tx_error_fatal = tx_error_fatal self.params = {} with open(keyfile, 'r') as f: self.priv_key = w3.eth.account.decrypt(f.read(), password) self.account = w3.eth.account.from_key(self.priv_key).address logger.info('Using account: %s', self.account) self.rate_limit = None # Do not init nonce manager here. Need to wait until we can guarantee that our event loop is set. self.nonce_managers = {} self.__schedules = {} self.tries = 0 self.bounties = None self.staking = None self.offers = None self.relay = None self.balances = None # Setup a liveliness instance self.liveness_recorder = LocalLivenessRecorder() # Events from client self.on_run = events.OnRunCallback() self.on_stop = events.OnStopCallback() # Events from polyswarmd self.on_new_block = events.OnNewBlockCallback() self.on_new_bounty = events.OnNewBountyCallback() self.on_new_assertion = events.OnNewAssertionCallback() self.on_reveal_assertion = events.OnRevealAssertionCallback() self.on_new_vote = events.OnNewVoteCallback() self.on_quorum_reached = events.OnQuorumReachedCallback() self.on_settled_bounty = events.OnSettledBountyCallback() self.on_initialized_channel = events.OnInitializedChannelCallback() self.on_deprecated = events.OnDeprecatedCallback() # Events scheduled on block deadlines self.on_reveal_assertion_due = events.OnRevealAssertionDueCallback() self.on_vote_on_bounty_due = events.OnVoteOnBountyDueCallback() self.on_settle_bounty_due = events.OnSettleBountyDueCallback() self.on_withdraw_stake_due = events.OnWithdrawStakeDueCallback() utils.configure_event_loop() def run(self, chains=None): """Run the main event loop Args: chains (set(str)): Set of chains to operate on. Defaults to {'home', 'side'} """ if chains is None: chains = {'home', 'side'} # noinspection PyBroadException try: asyncio.get_event_loop().run_until_complete(self.run_task(chains=chains)) except asyncio.CancelledError: logger.info('Clean exit requested') utils.asyncio_join() except Exception: logger.exception('Unhandled exception at top level') utils.asyncio_stop() utils.asyncio_join() async def run_task(self, chains=None, listen_for_events=True): """ How the event loop handles running a task. Args: chains (set(str)): Set of chains to operate on. Defaults to {'home', 'side'} listen_for_events (bool): Whether or not to listen to the websocket for events """ self.params = {'account': self.account} if chains is None: chains = {'home', 'side'} self.__schedules = {chain: events.Schedule() for chain in chains} # We can now create our locks, because we are assured that the event loop is set self.nonce_managers = {chain: NonceManager(self, chain) for chain in chains} for nonce_manager in self.nonce_managers.values(): await nonce_manager.setup() self.rate_limit = await RequestRateLimit.build() try: await self.liveness_recorder.start() await self.create_sub_clients(chains) for chain in chains: await self.bounties.fetch_parameters(chain) await self.staking.fetch_parameters(chain) await self.on_run.run(chain) # At this point we're initialized, reset our failure counter and listen for events self.tries = 0 if listen_for_events: await asyncio.wait([self.listen_for_events(chain) for chain in chains]) finally: await self.on_stop.run() self.clear_sub_clients() def clear_sub_clients(self): self.balances = None self.bounties = None self.offers = None self.relay = None self.staking = None @backoff.on_exception(backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_tries=3) async def create_sub_clients(self, chains): # Test polyswarmd, then either load ethereum or fast try: # Wallets only exists in polyswarmd-fast headers = {} if self.api_key: headers.update({'Authorization': self.api_key}) async with aiohttp.ClientSession() as session: async with session.options(f'{self.polyswarmd_uri}/wallets/', headers=headers) as response: if response.status == 404: logger.debug('Using ethereum sub-clients') self.create_ethereum_sub_clients(chains) return response.raise_for_status() logger.debug('Using fast sub-clients') self.create_fast_sub_clients(chains) except aiohttp.ClientConnectionError: logger.exception('Unable to connect to polyswarmd') raise except asyncio.TimeoutError: logger.exception('Timeout connecting to polyswarmd') raise def create_ethereum_sub_clients(self, chains): from polyswarmclient.ethereum import BalanceClient, BountiesClient, StakingClient, OffersClient, RelayClient self.bounties = BountiesClient(self) self.staking = StakingClient(self) self.offers = OffersClient(self) self.relay = RelayClient(self) self.balances = BalanceClient(self) def create_fast_sub_clients(self, chains): from polyswarmclient.fast import BalanceClient, BountiesClient, StakingClient, OffersClient, RelayClient self.bounties = BountiesClient(self) self.staking = StakingClient(self) self.offers = OffersClient(self) self.relay = RelayClient(self) self.balances = BalanceClient(self) async def periodic(): # FIXME PSC continues to hit a down polyswarmd, because the trigger is time, not blocks from websocket while True: number = int(math.floor(time.time())) for chain in chains: asyncio.get_event_loop().create_task(self.__handle_scheduled_events(number, chain=chain)) asyncio.get_event_loop().create_task(self.on_new_block.run(number=number, chain=chain)) asyncio.get_event_loop().create_task(self.liveness_recorder.advance_time(number)) await asyncio.sleep(1) asyncio.get_event_loop().create_task(periodic()) @utils.return_on_exception((aiohttp.ServerDisconnectedError, asyncio.TimeoutError, aiohttp.ClientOSError, aiohttp.ContentTypeError, RateLimitedError), default=(False, {})) async def make_request(self, method, path, chain, json=None, send_nonce=False, api_key=None, params=None): """Make a request to polyswarmd, expecting a json response Args: method (str): HTTP method to use path (str): Path portion of URI to send request to chain (str): Which chain to operate on json (obj): JSON payload to send with request send_nonce (bool): Whether to include a base_nonce query string parameter in this request api_key (str): Override default API key params (dict): Optional params for the request Returns: (bool, obj): Tuple of boolean representing success, and response JSON parsed from polyswarmd """ if chain != 'home' and chain != 'side': raise ValueError(f'Chain parameter must be `home` or `side`, got {chain}') uri = f'{self.polyswarmd_uri}{path}' logger.debug('making request to url: %s', uri) params = params or {} params.update(dict(self.params)) params['chain'] = chain if send_nonce: # Set to 0 because I will replace it later params['base_nonce'] = 0 # Allow overriding API key per request api_key = api_key or self.api_key headers = {} if api_key: headers = {'Authorization': api_key} response = {} try: await self.rate_limit.check() async with aiohttp.ClientSession() as session: async with session.request(method, uri, params=params, headers=headers, json=json) as raw: self._check_status_for_rate_limit(raw.status) try: response = await raw.json() except aiohttp.ContentTypeError: response = await raw.read() if raw else 'None' raise queries = '&'.join([a + '=' + str(b) for (a, b) in params.items()]) logger.debug('%s %s?%s', method, path, queries, extra={'extra': response}) if not utils.check_response(response): logger.warning('Request %s %s?%s failed', method, path, queries) return False, response.get('errors') return True, response.get('result') except aiohttp.ContentTypeError: logger.exception('Received non-json response from polyswarmd: %s, url: %s', response, uri) raise except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError): logger.exception('Connection to polyswarmd refused') raise except asyncio.TimeoutError: logger.error('Connection to polyswarmd timed out') raise except RateLimitedError: # Handle "Too many requests" rate limit by not hammering server, and pausing all requests for a bit logger.warning('Hit polyswarmd rate limits, stopping all requests for a moment') asyncio.get_event_loop().create_task(self.rate_limit.trigger()) raise async def list_artifacts(self, ipfs_uri, api_key=None): """Return a list of artifacts from a given ipfs_uri. Args: ipfs_uri (str): IPFS URI to get artifiacts from. api_key (str): Override default API key Returns: List[(str, str)]: A list of tuples. First tuple element is the artifact name, second tuple element is the artifact hash. """ if not utils.is_valid_uri(ipfs_uri): logger.warning('Invalid IPFS URI: %s', ipfs_uri) return [] path = f'/artifacts/{ipfs_uri}/' # Chain parameter doesn't matter for artifacts, just set to side success, result = await self.make_request('GET', path, 'side', api_key=api_key) if not success: logger.error('Expected artifact listing, received', extra={'extra': result}) return [] result = {} if result is None else result return [(a.get('name', ''), a.get('hash', '')) for a in result] async def get_artifact_count(self, ipfs_uri, api_key=None): """Gets the number of artifacts at the ipfs uri Args: ipfs_uri (str): IPFS URI for the artifact set api_key (str): Override default API key Returns: Number of artifacts at the uri """ artifacts = await self.list_artifacts(ipfs_uri, api_key=api_key) return len(artifacts) if artifacts is not None and artifacts else 0 @utils.return_on_exception((aiohttp.ServerDisconnectedError, asyncio.TimeoutError, aiohttp.ContentTypeError, RateLimitedError, aiohttp.ClientOSError), default=None) async def get_artifact(self, ipfs_uri, index, api_key=None): """Retrieve an artifact from IPFS via polyswarmd Args: ipfs_uri (str): IPFS hash of the artifact to retrieve index (int): Index of the sub artifact to retrieve api_key (str): Override default API key Returns: (bytes): Content of the artifact """ if not utils.is_valid_uri(ipfs_uri): raise ValueError('Invalid IPFS URI') uri = f'{self.polyswarmd_uri}/artifacts/{ipfs_uri}/{index}/' logger.debug('getting artifact from uri: %s', uri) params = dict(self.params) # Allow overriding API key per request api_key = api_key or self.api_key headers = {} if api_key: headers = {'Authorization': api_key} try: await self.rate_limit.check() async with aiohttp.ClientSession() as session: async with session.get(uri, params=params, headers=headers) as raw_response: # Handle "Too many requests" rate limit by not hammering server, and instead sleeping a bit self._check_status_for_rate_limit(raw_response.status) if raw_response.status / 100 == 2: return await raw_response.read() except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError): logger.exception('Connection to polyswarmd refused') raise except asyncio.TimeoutError: logger.error('Connection to polyswarmd timed out') raise except RateLimitedError: # Handle "Too many requests" rate limit by not hammering server, and pausing all requests for a bit logger.warning('Hit polyswarmd rate limits, stopping all requests for a moment') asyncio.get_event_loop().create_task(self.rate_limit.trigger()) raise @staticmethod def to_wei(amount, unit='ether'): return w3.toWei(amount, unit) @staticmethod def from_wei(amount, unit='ether'): return w3.fromWei(amount, unit) @utils.return_on_exception((aiohttp.ServerDisconnectedError, asyncio.TimeoutError, RateLimitedError, aiohttp.ClientOSError), default=None) async def post_artifacts(self, files, api_key=None): """Post artifacts to polyswarmd, flexible files parameter to support different use-cases Args: files (list[(filename, contents)]): The artifacts to upload, accepts one of: (filename, bytes): File name and contents to upload (filename, file_obj): (Optional) file name and file object to upload (filename, None): File name to open and upload api_key (str): Override default API key Returns: (str): IPFS URI of the uploaded artifact """ uri = f'{self.polyswarmd_uri}/artifacts/' logger.debug('posting artifact to uri: %s', uri) params = dict(self.params) # Allow overriding API key per request if api_key is None: api_key = self.api_key headers = {'Authorization': api_key} if api_key is not None else None # MultipartWriter can only be used once, recreate if on retry with aiohttp.MultipartWriter('form-data') as mpwriter: response = {} to_close = [] try: for filename, f in files: # If contents is None, open filename for reading and remember to close it if f is None: f = open(filename, 'rb') to_close.append(f) # If filename is None and our file object has a name attribute, use it if filename is None and hasattr(f, 'name'): filename = f.name if filename: filename = os.path.basename(filename) else: filename = 'file' payload = aiohttp.payload.get_payload(f, content_type='application/octet-stream') payload.set_content_disposition('form-data', name='file', filename=filename) mpwriter.append_payload(payload) await self.rate_limit.check() # Make the request async with aiohttp.ClientSession() as session: async with session.post(uri, params=params, headers=headers, data=mpwriter) as raw_response: self._check_status_for_rate_limit(raw_response.status) try: response = await raw_response.json() except (ValueError, aiohttp.ContentTypeError): response = await raw_response.read() if raw_response else 'None' logger.error('Received non-json response from polyswarmd: %s, uri: %s', response, uri) response = {} except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError): logger.exception('Connection to polyswarmd refused, files: %s', files) raise except asyncio.TimeoutError: logger.error('Connection to polyswarmd timed out, files: %s', files) raise except RateLimitedError: # Handle "Too many requests" rate limit by not hammering server, and pausing all requests for a bit logger.warning('Hit polyswarmd rate limits, stopping all requests for a moment') asyncio.get_event_loop().create_task(self.rate_limit.trigger()) raise finally: for f in to_close: f.close() logger.debug('POST/artifacts', extra={'extra': response}) if not utils.check_response(response): logger.info('Posting artifacts to polyswarmd failed, giving up') return None return response.get('result') @staticmethod def _check_status_for_rate_limit(status): if status == 429: raise RateLimitedError def schedule(self, expiration, event, chain): """Schedule an event to execute on a particular block Args: expiration (int): Which block to execute on event (Event): Event to trigger on expiration block chain (str): Which chain to operate on """ if chain != 'home' and chain != 'side': raise ValueError(f'Chain parameter must be `home` or `side`, got {chain}') self.__schedules[chain].put(expiration, event) async def __handle_scheduled_events(self, number, chain): """Perform scheduled events when a new block is reported Args: number (int): The current block number reported from polyswarmd chain (str): Which chain to operate on """ if chain != 'home' and chain != 'side': raise ValueError('Chain parameter must be `home` or `side`, got {chain}') while self.__schedules[chain].peek() and self.__schedules[chain].peek()[0] < number: exp, task = self.__schedules[chain].get() if isinstance(task, events.RevealAssertion): asyncio.get_event_loop().create_task( self.on_reveal_assertion_due.run(bounty_guid=task.guid, index=task.index, nonce=task.nonce, verdicts=task.verdicts, metadata=task.metadata, chain=chain)) elif isinstance(task, events.SettleBounty): asyncio.get_event_loop().create_task( self.on_settle_bounty_due.run(bounty_guid=task.guid, chain=chain)) elif isinstance(task, events.VoteOnBounty): asyncio.get_event_loop().create_task( self.on_vote_on_bounty_due.run(bounty_guid=task.guid, votes=task.votes, valid_bloom=task.valid_bloom, chain=chain)) elif isinstance(task, events.WithdrawStake): asyncio.get_event_loop().create_task( self.on_withdraw_stake_due.run(amount=task.amount, chain=chain)) async def listen_for_events(self, chain): """Listen for events via websocket connection to polyswarmd Args: chain (str): Which chain to operate on """ if chain != 'home' and chain != 'side': raise ValueError(f'Chain parameter must be `home` or `side`, got {chain}') if not self.polyswarmd_uri.startswith('http'): raise ValueError(f'polyswarmd_uri protocol is not http or https, got {self.polyswarmd_uri}') # http:// -> ws://, https:// -> wss:// wsuri = f'{self.polyswarmd_uri.replace("http", "ws", 1)}/events/?chain={chain}' last_block = 0 async for message in self.websocket_events(wsuri, chain): next_block = await self.route_websocket_message(message, last_block, chain) if next_block: last_block = next_block # If the websocket closes naturally, exit this function logger.info('%s chain closed the websocket', chain) async def websocket_events(self, websocket_uri, chain): exponential_backoff = BackoffWrapper(backoff.expo, max_value=MAX_BACKOFF) while True: try: async with websockets.connect(websocket_uri) as websocket: # reset backoff because we got a connection exponential_backoff.reset() logger.error('Websocket connection to polyswarmd established') # Fetch parameters again here because we may have missed update events await self.bounties.fetch_parameters(chain) await self.staking.fetch_parameters(chain) while not websocket.closed: message = await websocket.recv() if message is not None: logger.debug('Received message on websocket', extra={'extra': message}) yield message except (websockets.exceptions.ConnectionClosed, asyncio.streams.IncompleteReadError): logger.error('Websocket connection to polyswarmd closed, retrying') await exponential_backoff.sleep() except (OSError, websockets.exceptions.InvalidHandshake): logger.error('Websocket connection to polyswarmd refused, retrying') await exponential_backoff.sleep() except asyncio.TimeoutError: logger.error('Websocket connection to polyswarmd timed out, retrying') await exponential_backoff.sleep() async def route_websocket_message(self, message, last_block, chain): try: message = json.loads(message) event = message.get('event') data = message.get('data') block_number = message.get('block_number') txhash = message.get('txhash') except json.JSONDecodeError: logger.error('Invalid event message from polyswarmd: %s', message) return if event != 'block': logger.info('Received %s on chain %s', event, chain, extra={'extra': data}) if event == 'connected': logger.info('Connected to event socket at: %s', data.get('start_time')) elif event == 'block': number = data.get('number', 0) if number <= last_block: return if number % 100 == 0: logger.debug('Block %s on chain %s', number, chain) asyncio.get_event_loop().create_task(self.on_new_block.run(number=number, chain=chain)) # These are staying here because we need the homechain block events as well asyncio.get_event_loop().create_task(self.__handle_scheduled_events(number, chain=chain)) asyncio.get_event_loop().create_task(self.liveness_recorder.advance_time(number)) return number elif event == 'fee_update': d = {'bounty_fee': data.get('bounty_fee'), 'assertion_fee': data.get('assertion_fee')} await self.bounties.parameters[chain].update({k: v for k, v in d.items() if v is not None}) elif event == 'window_update': d = {'assertion_reveal_window': data.get('assertion_reveal_window'), 'arbiter_vote_window': data.get('arbiter_vote_window')} await self.bounties.parameters[chain].update({k: v for k, v in d.items() if v is not None}) elif event == 'bounty': asyncio.get_event_loop().create_task( self.on_new_bounty.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'assertion': asyncio.get_event_loop().create_task( self.on_new_assertion.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'reveal': asyncio.get_event_loop().create_task( self.on_reveal_assertion.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'vote': asyncio.get_event_loop().create_task( self.on_new_vote.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'quorum': asyncio.get_event_loop().create_task( self.on_quorum_reached.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'settled_bounty': asyncio.get_event_loop().create_task( self.on_settled_bounty.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'deprecated': asyncio.get_event_loop().create_task( self.on_deprecated.run(**data, block_number=block_number, txhash=txhash, chain=chain)) elif event == 'initialized_channel': asyncio.get_event_loop().create_task( self.on_initialized_channel.run(**data, block_number=block_number, txhash=txhash)) else: logger.error('Invalid event type from polyswarmd: %s', message)