def requestUrl(self, url, destination=None, params=None, data=None, usePost=False): result = RequestResult() if self._OAUTH is not None: # Use OAuth path try: result = self._OAUTH.request(url, destination, params, data=data, usePost=usePost) except (RequestExpiredToken, RequestInvalidToken): logging.exception('Cannot fetch due to token issues') result = RequestResult().setResult(RequestResult.OAUTH_INVALID) self.invalidateOAuth() except requests.exceptions.RequestException: logging.exception('request to download image failed') result = RequestResult().setResult(RequestResult.NO_NETWORK) else: tries = 0 while tries < 5: try: if usePost: r = requests.post(url, params=params, json=data, timeout=180) else: r = requests.get(url, params=params, timeout=180) break except: logging.exception('Issues downloading') time.sleep(tries * 10) # Back off 10, 20, ... depending on tries tries += 1 logging.warning('Retrying again, attempt #%d', tries) if tries == 5: logging.error('Failed to download due to network issues') raise RequestNoNetwork if r: result.setHTTPCode(r.status_code).setHeaders( r.headers).setResult(RequestResult.SUCCESS) if destination is None: result.setContent(r.content) else: with open(destination, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): f.write(chunk) result.setFilename(destination) return result
def requestUrl(self, url, destination=None, params=None, data=None, usePost=False): result = RequestResult() if self._OAUTH is not None: # Use OAuth path result = self._OAUTH.request(url, destination, params, data=data, usePost=usePost) else: tries = 0 while tries < 5: try: if usePost: r = requests.post(url, params=params, json=data) else: r = requests.get(url, params=params) break except: logging.exception('Issues downloading') time.sleep(tries / 10) # Back off 10, 20, ... depending on tries tries += 1 logging.warning('Retrying again, attempt #%d', tries) if tries == 5: logging.error('Failed to download due to network issues') raise RequestNoNetwork if r: result.setHTTPCode(r.status_code).setHeaders( r.headers).setResult(RequestResult.SUCCESS) if destination is None: result.setContent(r.content) else: with open(destination, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): f.write(chunk) result.setFilename(destination) return result
def request(self, uri, destination=None, params=None, data=None, usePost=False): ret = RequestResult() result = None stream = destination != None tries = 0 while tries < 5: try: try: auth = self.getSession() if auth is None: logging.error( 'Unable to get OAuth session, probably expired') raise RequestExpiredToken if usePost: result = auth.post(uri, stream=stream, params=params, json=data, timeout=180) else: result = auth.get(uri, stream=stream, params=params, timeout=180) if result is not None: break except TokenExpiredError: auth = self.getSession(True) if auth is None: logging.error( 'Unable to get OAuth session, probably expired') raise RequestExpiredToken if usePost: result = auth.post(uri, stream=stream, params=params, json=data, timeout=180) else: result = auth.get(uri, stream=stream, params=params, timeout=180) if result is not None: break except InvalidGrantError: logging.error( 'Token is no longer valid, need to re-authenticate') raise RequestInvalidToken except: logging.exception('Issues downloading') time.sleep(tries * 10) # Back off 10, 20, ... depending on tries tries += 1 logging.warning('Retrying again, attempt #%d', tries) if tries == 5: logging.error('Failed to download due to network issues') raise RequestNoNetwork if destination is not None: try: with open(destination, 'wb') as handle: for chunk in result.iter_content(chunk_size=512): if chunk: # filter out keep-alive new chunks handle.write(chunk) ret.setResult(RequestResult.SUCCESS).setHTTPCode( result.status_code) ret.setHeaders(result.headers) except: logging.exception('Failed to download %s' % uri) ret.setResult(RequestResult.FAILED_SAVING) else: ret.setResult(RequestResult.SUCCESS).setHTTPCode( result.status_code) ret.setHeaders(result.headers) ret.setContent(result.content) return ret