async def make_request(self,
method,
path,
chain,
json=None,
send_nonce=False,
api_key=None,
tries=2,
params=None):
"""Make a request to polyswarmd, expecting a json response
Args:
method (str): HTTP method to use
path (str): Path portion of URI to send request to
chain (str): Which chain to operate on
json (obj): JSON payload to send with request
send_nonce (bool): Whether to include a base_nonce query string parameter in this request
api_key (str): Override default API key
tries (int): Number of times to retry before giving up
params (dict): Optional params for the request
Returns:
(bool, obj): Tuple of boolean representing success, and response JSON parsed from polyswarmd
"""
if chain != 'home' and chain != 'side':
raise ValueError(
'Chain parameter must be `home` or `side`, got {0}'.format(
chain))
if self.__session is None or self.__session.closed:
raise Exception('Not running')
# Ensure we try at least once
tries = max(tries, 1)
uri = f'{self.polyswarmd_uri}{path}'
logger.debug('making request to url: %s', uri)
if params is None:
params = dict()
params.update(dict(self.params))
params['chain'] = chain
if send_nonce:
# Set to 0 because I will replace it later
params['base_nonce'] = 0
# Allow overriding API key per request
if api_key is None:
api_key = self.api_key
headers = {'Authorization': api_key} if api_key is not None else None
qs = '&'.join([a + '=' + str(b) for (a, b) in params.items()])
response = {}
while tries > 0:
tries -= 1
response = {}
try:
async with self.__session.request(method,
uri,
params=params,
headers=headers,
json=json) as raw_response:
try:
# Handle "Too many requests" rate limit by not hammering server, and instead sleeping a bit
if raw_response.status == 429:
logger.warning(
'Hit polyswarmd rate limits, sleeping then trying again'
)
await asyncio.sleep(RATE_LIMIT_SLEEP)
tries += 1
continue
response = await raw_response.json()
except (ValueError, aiohttp.ContentTypeError):
response = await raw_response.read(
) if raw_response else 'None'
logger.error(
'Received non-json response from polyswarmd: %s, url: %s',
response, uri)
response = {}
continue
except (OSError, aiohttp.ServerDisconnectedError):
logger.error('Connection to polyswarmd refused, retrying')
except asyncio.TimeoutError:
logger.error('Connection to polyswarmd timed out, retrying')
logger.debug('%s %s?%s',
method,
path,
qs,
extra={'extra': response})
if not check_response(response):
if tries > 0:
logger.info('Request %s %s?%s failed, retrying...', method,
path, qs)
continue
else:
logger.warning('Request %s %s?%s failed, giving up',
method, path, qs)
return False, response.get('errors')
return True, response.get('result')
return False, response.get('errors')
async def post_artifacts(self, files, api_key=None, tries=2):
"""Post artifacts to polyswarmd, flexible files parameter to support different use-cases
Args:
files (list[(filename, contents)]): The artifacts to upload, accepts one of:
(filename, bytes): File name and contents to upload
(filename, file_obj): (Optional) file name and file object to upload
(filename, None): File name to open and upload
api_key (str): Override default API key
Returns:
(str): IPFS URI of the uploaded artifact
"""
uri = f'{self.polyswarmd_uri}/artifacts'
logger.debug('posting artifact to uri: %s', uri)
params = dict(self.params)
# Allow overriding API key per request
if api_key is None:
api_key = self.api_key
headers = {'Authorization': api_key} if api_key is not None else None
while tries > 0:
tries -= 1
# MultipartWriter can only be used once, recreate if on retry
with aiohttp.MultipartWriter('form-data') as mpwriter:
response = {}
to_close = []
try:
for filename, f in files:
# If contents is None, open filename for reading and remember to close it
if f is None:
f = open(filename, 'rb')
to_close.append(f)
# If filename is None and our file object has a name attribute, use it
if filename is None and hasattr(f, 'name'):
filename = f.name
if filename:
filename = os.path.basename(filename)
else:
filename = 'file'
payload = aiohttp.payload.get_payload(
f, content_type='application/octet-stream')
payload.set_content_disposition('form-data',
name='file',
filename=filename)
mpwriter.append_payload(payload)
# Make the request
async with self.__session.post(
uri, params=params, headers=headers,
data=mpwriter) as raw_response:
try:
# Handle "Too many requests" rate limit by not hammering server, and instead sleeping a bit
if raw_response.status == 429:
logger.warning(
'Hit polyswarmd rate limits, sleeping then trying again'
)
await asyncio.sleep(RATE_LIMIT_SLEEP)
tries += 1
continue
response = await raw_response.json()
except (ValueError, aiohttp.ContentTypeError):
response = await raw_response.read(
) if raw_response else 'None'
logger.error(
'Received non-json response from polyswarmd: %s, uri: %s',
response, uri)
response = {}
continue
except (OSError, aiohttp.ServerDisconnectedError):
logger.error('Connection to polyswarmd refused, files: %s',
files)
except asyncio.TimeoutError:
logger.error(
'Connection to polyswarmd timed out, files: %s', files)
finally:
for f in to_close:
f.close()
logger.debug('POST/artifacts', extra={'extra': response})
if not check_response(response):
if tries > 0:
logger.info(
'Posting artifacts to polyswarmd failed, retrying')
continue
else:
logger.info(
'Posting artifacts to polyswarmd failed, giving up'
)
return None
return response.get('result')
async def make_request(self, method, path, chain, json=None, send_nonce=False, api_key=None, params=None):
"""Make a request to polyswarmd, expecting a json response
Args:
method (str): HTTP method to use
path (str): Path portion of URI to send request to
chain (str): Which chain to operate on
json (obj): JSON payload to send with request
send_nonce (bool): Whether to include a base_nonce query string parameter in this request
api_key (str): Override default API key
params (dict): Optional params for the request
Returns:
(bool, obj): Tuple of boolean representing success, and response JSON parsed from polyswarmd
"""
if chain != 'home' and chain != 'side':
raise ValueError(f'Chain parameter must be `home` or `side`, got {chain}')
uri = f'{self.polyswarmd_uri}{path}'
logger.debug('making request to url: %s', uri)
params = params or {}
params.update(dict(self.params))
params['chain'] = chain
if send_nonce:
# Set to 0 because I will replace it later
params['base_nonce'] = 0
# Allow overriding API key per request
api_key = api_key or self.api_key
headers = {}
if api_key:
headers = {'Authorization': api_key}
response = {}
try:
await self.rate_limit.check()
async with aiohttp.ClientSession() as session:
async with session.request(method, uri, params=params, headers=headers, json=json) as raw:
self._check_status_for_rate_limit(raw.status)
try:
response = await raw.json()
except aiohttp.ContentTypeError:
response = await raw.read() if raw else 'None'
raise
queries = '&'.join([a + '=' + str(b) for (a, b) in params.items()])
logger.debug('%s %s?%s', method, path, queries, extra={'extra': response})
if not utils.check_response(response):
logger.warning('Request %s %s?%s failed', method, path, queries)
return False, response.get('errors')
return True, response.get('result')
except aiohttp.ContentTypeError:
logger.exception('Received non-json response from polyswarmd: %s, url: %s', response, uri)
raise
except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
logger.exception('Connection to polyswarmd refused')
raise
except asyncio.TimeoutError:
logger.error('Connection to polyswarmd timed out')
raise
except RateLimitedError:
# Handle "Too many requests" rate limit by not hammering server, and pausing all requests for a bit
logger.warning('Hit polyswarmd rate limits, stopping all requests for a moment')
asyncio.get_event_loop().create_task(self.rate_limit.trigger())
raise