req = urllib2.Request(url, headers={'User-agent': settings.USER_AGENT})
try:
request = urllib2.urlopen(req, timeout=settings.HTTP_TIMEOUT)
except urllib2.HTTPError, e:
logger.error('Failed to obtain %s : Status %s' % (url, e.code))
except urllib2.URLError, e:
if 'timed out' in str(e):
# On a timeout, retry in hopes that it won't next time
self.retry(args=[url], exc=e)
else:
logger.exception('Unexeped error when retrieving OEmbed %s' % url)
else:
if request.getcode() != 200:
logger.error('URL %s resulted in unexpected HTTP status' % (url, request.getcode()))
else:
original_url = extract_content_url(url)
try:
# TODO: Any validation that should happen here?
# Do we store invalid data? If invalid do we clear the cache?
data = json.loads(request.read())
except ValueError:
logger.error('OEmbed response from %s contains invalid JSON' % url)
else:
# Update the cache with this data
cache.set(url, Resource(original_url, data))
finally:
request.close()
request_external_oembed = registry.tasks[RequestExternalOEmbedTask.name]