def fetchImage(self, image, destinationDir, supportedMimeTypes, displaySize): filename = os.path.join(destinationDir, self.generateFilename()) if image.cacheAllow: # Look it up in the cache mgr if self._CACHEMGR is None: logging.error('CacheManager is not available') else: cacheFile = self._CACHEMGR.getCachedImage( image.getCacheId(), filename) if cacheFile: image.setFilename(cacheFile) image.cacheUsed = True if not image.cacheUsed: recommendedSize = self.calcRecommendedSize(image.dimensions, displaySize) if recommendedSize is None: recommendedSize = displaySize url = self.getContentUrl(image, { 'size': recommendedSize, 'display': displaySize }) if url is None: return ImageHolder().setError( 'Unable to download image, no URL') try: result = self.requestUrl(url, destination=filename) except (RequestResult.RequestExpiredToken, RequestInvalidToken): logging.exception('Cannot fetch due to token issues') result = RequestResult().setResult(RequestResult.OAUTH_INVALID) self._OAUTH = None except requests.exceptions.RequestException: logging.exception('request to download image failed') result = RequestResult().setResult(RequestResult.NO_NETWORK) if not result.isSuccess(): return ImageHolder().setError('%d: Unable to download image!' % result.httpcode) else: image.setFilename(filename) if image.filename is not None: image.setMimetype(helper.getMimetype(image.filename)) return image
def requestUrl(self, url, destination=None, params=None, data=None, usePost=False): result = RequestResult() if self._OAUTH is not None: # Use OAuth path result = self._OAUTH.request(url, destination, params, data=data, usePost=usePost) else: tries = 0 while tries < 5: try: if usePost: r = requests.post(url, params=params, json=data, timeout=180) else: r = requests.get(url, params=params, timeout=180) break except: logging.exception('Issues downloading') time.sleep(tries / 10) # Back off 10, 20, ... depending on tries tries += 1 logging.warning('Retrying again, attempt #%d', tries) if tries == 5: logging.error('Failed to download due to network issues') raise RequestNoNetwork if r: result.setHTTPCode(r.status_code).setHeaders( r.headers).setResult(RequestResult.SUCCESS) if destination is None: result.setContent(r.content) else: with open(destination, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): f.write(chunk) result.setFilename(destination) return result
def requestUrl(self, url, destination=None, params=None, data=None, usePost=False): # pretend to download the file (for compatability with 'selectImageFromAlbum' of baseService) # instead just cache a scaled version of the file and return {status: 200} result = RequestResult() if url.count("||") == 2: filename, width, height = url.split("||", 2) recSize = {"width": width, "height": height} else: filename = url recSize = None if destination is None or not os.path.isfile(filename): result.setResult(RequestResult.SUCCESS).setHTTPCode(400) elif recSize is not None and helper.scaleImage(filename, destination, recSize): result.setFilename(destination) result.setResult(RequestResult.SUCCESS).setHTTPCode(200) elif helper.copyFile(filename, destination): result.setFilename(destination) result.setResult(RequestResult.SUCCESS).setHTTPCode(200) else: result.setResult(RequestResult.SUCCESS).setHTTPCode(418) return result
def request(self, uri, destination=None, params=None, data=None, usePost=False): ret = RequestResult() result = None stream = destination != None tries = 0 while tries < 5: try: try: auth = self.getSession() if auth is None: logging.error( 'Unable to get OAuth session, probably expired') raise RequestExpiredToken if usePost: result = auth.post(uri, stream=stream, params=params, json=data, timeout=180) else: result = auth.get(uri, stream=stream, params=params, timeout=180) if result is not None: break except TokenExpiredError: auth = self.getSession(True) if auth is None: logging.error( 'Unable to get OAuth session, probably expired') raise RequestExpiredToken if usePost: result = auth.post(uri, stream=stream, params=params, json=data, timeout=180) else: result = auth.get(uri, stream=stream, params=params, timeout=180) if result is not None: break except InvalidGrantError: logging.error( 'Token is no longer valid, need to re-authenticate') raise RequestInvalidToken except: logging.exception('Issues downloading') time.sleep(tries * 10) # Back off 10, 20, ... depending on tries tries += 1 logging.warning('Retrying again, attempt #%d', tries) if tries == 5: logging.error('Failed to download due to network issues') raise RequestNoNetwork if destination is not None: try: with open(destination, 'wb') as handle: for chunk in result.iter_content(chunk_size=512): if chunk: # filter out keep-alive new chunks handle.write(chunk) ret.setResult(RequestResult.SUCCESS).setHTTPCode( result.status_code) ret.setHeaders(result.headers) except: logging.exception('Failed to download %s' % uri) ret.setResult(RequestResult.FAILED_SAVING) else: ret.setResult(RequestResult.SUCCESS).setHTTPCode( result.status_code) ret.setHeaders(result.headers) ret.setContent(result.content) return ret
def selectImageFromAlbum(self, destinationDir, supportedMimeTypes, displaySize, randomize): # chooses an album and selects an image from that album --> return {'id':, 'mimetype':, 'error':, 'source':} # if no (new) images can be found --> return None keywordList = list(self.getKeywords()) keywordCount = len(keywordList) if keywordCount == 0: return ImageHolder().setError('No albums have been specified') if randomize: index = self.getRandomKeywordIndex() else: index = self.keywordIndex # if current keywordList[index] does not contain any new images --> just run through all albums for i in range(0, keywordCount): if not randomize and (index + i) >= keywordCount: # (non-random image order): return if the last album is exceeded --> serviceManager should use next service break self.keywordIndex = (index + i) % keywordCount keyword = keywordList[self.keywordIndex] # a provider-specific implementation for 'getImagesFor' is obligatory! images = self.getImagesFor(keyword) if images is None: logging.warning( 'Function returned None, this is used sometimes when a temporary error happens. Still logged' ) self.imageIndex = 0 continue if len(images) > 0 and images[0].error is not None: return images[0] self._STATE["_NUM_IMAGES"][keyword] = len(images) if len(images) == 0: self.imageIndex = 0 continue self.saveState() image = self.selectImage(images, supportedMimeTypes, displaySize, randomize) if image is None: self.imageIndex = 0 continue filename = os.path.join(destinationDir, image.id) # you should implement 'addUrlParams' if the provider allows 'width' / 'height' parameters! recommendedSize = self.calcRecommendedSize(image.dimensions, displaySize) url = self.addUrlParams(image.url, recommendedSize, displaySize) if image.cacheAllow: # Look it up in the cache mgr if self._CACHEMGR is None: logging.error('CacheManager is not available') else: cacheFile = self._CACHEMGR.getCachedImage( image.getCacheId(), filename) if cacheFile: image.setFilename(cacheFile) image.cacheUsed = True if not image.cacheUsed: try: result = self.requestUrl(url, destination=filename) except requests.exceptions.RequestException: logging.exception('request to download image failed') result = RequestResult().setResult( RequestResult.NO_NETWORK) if not result.isSuccess(): return ImageHolder().setError( '%d: Unable to download image!' % result.httpcode) else: image.setFilename(filename) image.setMimetype(helper.getMimetype(image.filename)) return image self.resetIndices() return None