def urlopen(url): url = iri_to_uri(url) request = urllib.request.Request(url) request.add_header('Accept-Encoding', 'gzip') request.add_header('User-Agent', get_useragent()) while (time.time() - __reference_time) < __minimum_request_separation: time.sleep(__minimum_request_separation) __reference_time = time.time() try: data = urllib.request.urlopen(request).read() except urllib.error.URLError as e: try: msg = '%s (%s)' % (e.reason.strerror, e.reason.errno) except AttributeError: msg = str(e) raise RetrievalError(msg) except socket.error as e: msg = '%s (%s)' % (e.strerror, e.errno) raise RetrievalError(msg) except EnvironmentError as e: msg = '%s (%s)' % (e.strerror, e.errno) raise RetrievalError(msg) try: data = gzip.decompress(data) except IOError: "Gzipped data not returned." return data
def urlopen(url): url = iri_to_uri(url) request = urllib2.Request(url) request.add_header('Accept-Encoding', 'gzip') if puddlestuff.tagsources.user_agent: request.add_header('User-Agent', puddlestuff.tagsources.user_agent) else: request.add_header('User-Agent', 'puddetag/' + puddlestuff.version_string) if time.time() - __lasttime.time < 1: time.sleep(1) __lasttime.time = time.time() try: data = urllib2.urlopen(request).read() except urllib2.URLError, e: try: msg = u'%s (%s)' % (e.reason.strerror, e.reason.errno) except AttributeError: msg = unicode(e) raise RetrievalError(msg)
def urlopen(url): url = iri_to_uri(url) request = urllib2.Request(url) request.add_header('Accept-Encoding', 'gzip') request.add_header('User-Agent', get_useragent()) if time.time() - __lasttime.time < 1: time.sleep(1) __lasttime.time = time.time() try: data = urllib2.urlopen(request).read() except urllib2.URLError as e: try: msg = u'%s (%s)' % (e.reason.strerror, e.reason.errno) except AttributeError: msg = unicode(e) raise RetrievalError(msg) except socket.error as e: msg = u'%s (%s)' % (e.strerror, e.errno) raise RetrievalError(msg) except EnvironmentError, e: msg = u'%s (%s)' % (e.strerror, e.errno) raise RetrievalError(msg)
def search(album): search_url = create_search(album.replace('/', ' ')) write_log('Search URL - %s' % search_url) return urlopen(iri_to_uri(search_url))
def search(album): search_url = create_search(album.replace(u'/', u' ')) write_log(u'Search URL - %s' % search_url) return urlopen(iri_to_uri(search_url))