req = urllib2.Request(url, headers={'User-agent': settings.USER_AGENT}) try: request = urllib2.urlopen(req, timeout=settings.HTTP_TIMEOUT) except urllib2.HTTPError, e: logger.error('Failed to obtain %s : Status %s' % (url, e.code)) except urllib2.URLError, e: if 'timed out' in str(e): # On a timeout, retry in hopes that it won't next time self.retry(args=[url], exc=e) else: logger.exception('Unexeped error when retrieving OEmbed %s' % url) else: if request.getcode() != 200: logger.error('URL %s resulted in unexpected HTTP status' % (url, request.getcode())) else: original_url = extract_content_url(url) try: # TODO: Any validation that should happen here? # Do we store invalid data? If invalid do we clear the cache? data = json.loads(request.read()) except ValueError: logger.error('OEmbed response from %s contains invalid JSON' % url) else: # Update the cache with this data cache.set(url, Resource(original_url, data)) finally: request.close() request_external_oembed = registry.tasks[RequestExternalOEmbedTask.name]