def request_and_return(self): print "[SP] request_and_return webpage.." url = "http://www.wunschliste.de/updates/stationen" from plugin import buildURL, USER_AGENT req = Request(buildURL(url), headers={'User-Agent':USER_AGENT}) try: data = urlopen(req).read() except URLError as e: self.__errback(str(e)) except socket.timeout as e: self.__errback(str(e)) return self.__callback(data)
def getPage(self, url, use_proxy=True, counter=0): response = None logDebug("IB: getPage", url) cached = self.getCached(url) if cached: logDebug("IB: cached") response = cached else: logDebug("IB: not cached") try: from plugin import buildURL, USER_AGENT if use_proxy: temp_url = buildURL(url) else: temp_url = url req = Request(temp_url, headers={"User-Agent": USER_AGENT}) response = urlopen(req, timeout=float(config.plugins.seriesplugin.socket_timeout.value)).read() if not response: logDebug("IB: No data returned") # logDebug("IB: response to cache: ", response) # if response: # self.doCachePage(url, response) except URLError as e: # For Python 2.6 if counter > 2: logDebug("IB: URLError counter > 2") from SeriesPlugin import getInstance instance = getInstance() if instance: instance.stop() raise MyException("There was an URLError: %r" % e) elif hasattr(e, "code"): logDebug("IB: URLError code", e.code, e.msg, counter) sleep(2) return self.getPage(url, use_proxy, counter + 1) else: logDebug("IB: URLError else") raise MyException("There was an URLError: %r" % e) except socket.timeout as e: # For Python 2.7 if counter > 2: logDebug("IB: URLError counter > 2") from SeriesPlugin import getInstance instance = getInstance() if instance: instance.stop() raise MyException("There was an SocketTimeout: %r" % e) elif hasattr(e, "code"): logDebug("IB: URLError code", e.code, e.msg, counter) sleep(2) return self.getPage(url, use_proxy, counter + 1) else: logDebug("IB: URLError else") raise MyException("There was an SocketTimeout: %r" % e) logDebug("IB: success") return response
def request(self): print "[SP] request webpage.." url = "http://www.wunschliste.de/updates/stationen" from plugin import buildURL, USER_AGENT getPage( buildURL(url), headers={'User-Agent':USER_AGENT}).addCallback(self.__callback).addErrback(self.__errback)
def getPage(self, url, use_proxy=True, counter=0): response = None logDebug("IB: getPage", url) cached = self.getCached(url) if cached: logDebug("IB: cached") response = cached else: logDebug("IB: not cached") try: from plugin import buildURL, USER_AGENT if use_proxy: temp_url = buildURL(url) else: temp_url = url req = Request(temp_url, headers={'User-Agent': USER_AGENT}) response = urlopen(req, timeout=float(config.plugins.seriesplugin. socket_timeout.value)).read() if not response: logDebug("IB: No data returned") #logDebug("IB: response to cache: ", response) #if response: # self.doCachePage(url, response) except URLError as e: # For Python 2.6 if counter > 2: logDebug("IB: URLError counter > 2") from SeriesPlugin import getInstance instance = getInstance() if instance: instance.stop() raise MyException("There was an URLError: %r" % e) elif hasattr(e, "code"): logDebug("IB: URLError code", e.code, e.msg, counter) sleep(2) return self.getPage(url, use_proxy, counter + 1) else: logDebug("IB: URLError else") raise MyException("There was an URLError: %r" % e) except socket.timeout as e: # For Python 2.7 if counter > 2: logDebug("IB: URLError counter > 2") from SeriesPlugin import getInstance instance = getInstance() if instance: instance.stop() raise MyException("There was an SocketTimeout: %r" % e) elif hasattr(e, "code"): logDebug("IB: URLError code", e.code, e.msg, counter) sleep(2) return self.getPage(url, use_proxy, counter + 1) else: logDebug("IB: URLError else") raise MyException("There was an SocketTimeout: %r" % e) logDebug("IB: success") return response