def fetchImage(self, image, destinationDir, supportedMimeTypes,
displaySize):
filename = os.path.join(destinationDir, self.generateFilename())
if image.cacheAllow:
# Look it up in the cache mgr
if self._CACHEMGR is None:
logging.error('CacheManager is not available')
else:
cacheFile = self._CACHEMGR.getCachedImage(
image.getCacheId(), filename)
if cacheFile:
image.setFilename(cacheFile)
image.cacheUsed = True
if not image.cacheUsed:
recommendedSize = self.calcRecommendedSize(image.dimensions,
displaySize)
if recommendedSize is None:
recommendedSize = displaySize
url = self.getContentUrl(image, {
'size': recommendedSize,
'display': displaySize
})
if url is None:
return ImageHolder().setError(
'Unable to download image, no URL')
try:
result = self.requestUrl(url, destination=filename)
except (RequestResult.RequestExpiredToken, RequestInvalidToken):
logging.exception('Cannot fetch due to token issues')
result = RequestResult().setResult(RequestResult.OAUTH_INVALID)
self._OAUTH = None
except requests.exceptions.RequestException:
logging.exception('request to download image failed')
result = RequestResult().setResult(RequestResult.NO_NETWORK)
if not result.isSuccess():
return ImageHolder().setError('%d: Unable to download image!' %
result.httpcode)
else:
image.setFilename(filename)
if image.filename is not None:
image.setMimetype(helper.getMimetype(image.filename))
return image
def requestUrl(self,
url,
destination=None,
params=None,
data=None,
usePost=False):
result = RequestResult()
if self._OAUTH is not None:
# Use OAuth path
result = self._OAUTH.request(url,
destination,
params,
data=data,
usePost=usePost)
else:
tries = 0
while tries < 5:
try:
if usePost:
r = requests.post(url,
params=params,
json=data,
timeout=180)
else:
r = requests.get(url, params=params, timeout=180)
break
except:
logging.exception('Issues downloading')
time.sleep(tries /
10) # Back off 10, 20, ... depending on tries
tries += 1
logging.warning('Retrying again, attempt #%d', tries)
if tries == 5:
logging.error('Failed to download due to network issues')
raise RequestNoNetwork
if r:
result.setHTTPCode(r.status_code).setHeaders(
r.headers).setResult(RequestResult.SUCCESS)
if destination is None:
result.setContent(r.content)
else:
with open(destination, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
result.setFilename(destination)
return result
def requestUrl(self,
url,
destination=None,
params=None,
data=None,
usePost=False):
# pretend to download the file (for compatability with 'selectImageFromAlbum' of baseService)
# instead just cache a scaled version of the file and return {status: 200}
result = RequestResult()
if url.count("||") == 2:
filename, width, height = url.split("||", 2)
recSize = {"width": width, "height": height}
else:
filename = url
recSize = None
if destination is None or not os.path.isfile(filename):
result.setResult(RequestResult.SUCCESS).setHTTPCode(400)
elif recSize is not None and helper.scaleImage(filename, destination,
recSize):
result.setFilename(destination)
result.setResult(RequestResult.SUCCESS).setHTTPCode(200)
elif helper.copyFile(filename, destination):
result.setFilename(destination)
result.setResult(RequestResult.SUCCESS).setHTTPCode(200)
else:
result.setResult(RequestResult.SUCCESS).setHTTPCode(418)
return result
def request(self,
uri,
destination=None,
params=None,
data=None,
usePost=False):
ret = RequestResult()
result = None
stream = destination != None
tries = 0
while tries < 5:
try:
try:
auth = self.getSession()
if auth is None:
logging.error(
'Unable to get OAuth session, probably expired')
raise RequestExpiredToken
if usePost:
result = auth.post(uri,
stream=stream,
params=params,
json=data,
timeout=180)
else:
result = auth.get(uri,
stream=stream,
params=params,
timeout=180)
if result is not None:
break
except TokenExpiredError:
auth = self.getSession(True)
if auth is None:
logging.error(
'Unable to get OAuth session, probably expired')
raise RequestExpiredToken
if usePost:
result = auth.post(uri,
stream=stream,
params=params,
json=data,
timeout=180)
else:
result = auth.get(uri,
stream=stream,
params=params,
timeout=180)
if result is not None:
break
except InvalidGrantError:
logging.error(
'Token is no longer valid, need to re-authenticate')
raise RequestInvalidToken
except:
logging.exception('Issues downloading')
time.sleep(tries * 10) # Back off 10, 20, ... depending on tries
tries += 1
logging.warning('Retrying again, attempt #%d', tries)
if tries == 5:
logging.error('Failed to download due to network issues')
raise RequestNoNetwork
if destination is not None:
try:
with open(destination, 'wb') as handle:
for chunk in result.iter_content(chunk_size=512):
if chunk: # filter out keep-alive new chunks
handle.write(chunk)
ret.setResult(RequestResult.SUCCESS).setHTTPCode(
result.status_code)
ret.setHeaders(result.headers)
except:
logging.exception('Failed to download %s' % uri)
ret.setResult(RequestResult.FAILED_SAVING)
else:
ret.setResult(RequestResult.SUCCESS).setHTTPCode(
result.status_code)
ret.setHeaders(result.headers)
ret.setContent(result.content)
return ret
def selectImageFromAlbum(self, destinationDir, supportedMimeTypes,
displaySize, randomize):
# chooses an album and selects an image from that album --> return {'id':, 'mimetype':, 'error':, 'source':}
# if no (new) images can be found --> return None
keywordList = list(self.getKeywords())
keywordCount = len(keywordList)
if keywordCount == 0:
return ImageHolder().setError('No albums have been specified')
if randomize:
index = self.getRandomKeywordIndex()
else:
index = self.keywordIndex
# if current keywordList[index] does not contain any new images --> just run through all albums
for i in range(0, keywordCount):
if not randomize and (index + i) >= keywordCount:
# (non-random image order): return if the last album is exceeded --> serviceManager should use next service
break
self.keywordIndex = (index + i) % keywordCount
keyword = keywordList[self.keywordIndex]
# a provider-specific implementation for 'getImagesFor' is obligatory!
images = self.getImagesFor(keyword)
if images is None:
logging.warning(
'Function returned None, this is used sometimes when a temporary error happens. Still logged'
)
self.imageIndex = 0
continue
if len(images) > 0 and images[0].error is not None:
return images[0]
self._STATE["_NUM_IMAGES"][keyword] = len(images)
if len(images) == 0:
self.imageIndex = 0
continue
self.saveState()
image = self.selectImage(images, supportedMimeTypes, displaySize,
randomize)
if image is None:
self.imageIndex = 0
continue
filename = os.path.join(destinationDir, image.id)
# you should implement 'addUrlParams' if the provider allows 'width' / 'height' parameters!
recommendedSize = self.calcRecommendedSize(image.dimensions,
displaySize)
url = self.addUrlParams(image.url, recommendedSize, displaySize)
if image.cacheAllow:
# Look it up in the cache mgr
if self._CACHEMGR is None:
logging.error('CacheManager is not available')
else:
cacheFile = self._CACHEMGR.getCachedImage(
image.getCacheId(), filename)
if cacheFile:
image.setFilename(cacheFile)
image.cacheUsed = True
if not image.cacheUsed:
try:
result = self.requestUrl(url, destination=filename)
except requests.exceptions.RequestException:
logging.exception('request to download image failed')
result = RequestResult().setResult(
RequestResult.NO_NETWORK)
if not result.isSuccess():
return ImageHolder().setError(
'%d: Unable to download image!' % result.httpcode)
else:
image.setFilename(filename)
image.setMimetype(helper.getMimetype(image.filename))
return image
self.resetIndices()
return None