def requestUrl(self,
url,
destination=None,
params=None,
data=None,
usePost=False):
result = RequestResult()
if self._OAUTH is not None:
# Use OAuth path
try:
result = self._OAUTH.request(url,
destination,
params,
data=data,
usePost=usePost)
except (RequestExpiredToken, RequestInvalidToken):
logging.exception('Cannot fetch due to token issues')
result = RequestResult().setResult(RequestResult.OAUTH_INVALID)
self.invalidateOAuth()
except requests.exceptions.RequestException:
logging.exception('request to download image failed')
result = RequestResult().setResult(RequestResult.NO_NETWORK)
else:
tries = 0
while tries < 5:
try:
if usePost:
r = requests.post(url,
params=params,
json=data,
timeout=180)
else:
r = requests.get(url, params=params, timeout=180)
break
except:
logging.exception('Issues downloading')
time.sleep(tries *
10) # Back off 10, 20, ... depending on tries
tries += 1
logging.warning('Retrying again, attempt #%d', tries)
if tries == 5:
logging.error('Failed to download due to network issues')
raise RequestNoNetwork
if r:
result.setHTTPCode(r.status_code).setHeaders(
r.headers).setResult(RequestResult.SUCCESS)
if destination is None:
result.setContent(r.content)
else:
with open(destination, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
result.setFilename(destination)
return result
def requestUrl(self,
url,
destination=None,
params=None,
data=None,
usePost=False):
result = RequestResult()
if self._OAUTH is not None:
# Use OAuth path
result = self._OAUTH.request(url,
destination,
params,
data=data,
usePost=usePost)
else:
tries = 0
while tries < 5:
try:
if usePost:
r = requests.post(url, params=params, json=data)
else:
r = requests.get(url, params=params)
break
except:
logging.exception('Issues downloading')
time.sleep(tries /
10) # Back off 10, 20, ... depending on tries
tries += 1
logging.warning('Retrying again, attempt #%d', tries)
if tries == 5:
logging.error('Failed to download due to network issues')
raise RequestNoNetwork
if r:
result.setHTTPCode(r.status_code).setHeaders(
r.headers).setResult(RequestResult.SUCCESS)
if destination is None:
result.setContent(r.content)
else:
with open(destination, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
result.setFilename(destination)
return result
def request(self,
uri,
destination=None,
params=None,
data=None,
usePost=False):
ret = RequestResult()
result = None
stream = destination != None
tries = 0
while tries < 5:
try:
try:
auth = self.getSession()
if auth is None:
logging.error(
'Unable to get OAuth session, probably expired')
raise RequestExpiredToken
if usePost:
result = auth.post(uri,
stream=stream,
params=params,
json=data,
timeout=180)
else:
result = auth.get(uri,
stream=stream,
params=params,
timeout=180)
if result is not None:
break
except TokenExpiredError:
auth = self.getSession(True)
if auth is None:
logging.error(
'Unable to get OAuth session, probably expired')
raise RequestExpiredToken
if usePost:
result = auth.post(uri,
stream=stream,
params=params,
json=data,
timeout=180)
else:
result = auth.get(uri,
stream=stream,
params=params,
timeout=180)
if result is not None:
break
except InvalidGrantError:
logging.error(
'Token is no longer valid, need to re-authenticate')
raise RequestInvalidToken
except:
logging.exception('Issues downloading')
time.sleep(tries * 10) # Back off 10, 20, ... depending on tries
tries += 1
logging.warning('Retrying again, attempt #%d', tries)
if tries == 5:
logging.error('Failed to download due to network issues')
raise RequestNoNetwork
if destination is not None:
try:
with open(destination, 'wb') as handle:
for chunk in result.iter_content(chunk_size=512):
if chunk: # filter out keep-alive new chunks
handle.write(chunk)
ret.setResult(RequestResult.SUCCESS).setHTTPCode(
result.status_code)
ret.setHeaders(result.headers)
except:
logging.exception('Failed to download %s' % uri)
ret.setResult(RequestResult.FAILED_SAVING)
else:
ret.setResult(RequestResult.SUCCESS).setHTTPCode(
result.status_code)
ret.setHeaders(result.headers)
ret.setContent(result.content)
return ret