def __init__(self, url, baseurl=None): self.url = url self.web_view = QWebView() self.network_manager = SplashQNetworkAccessManager() self.web_page = SplashQWebPage() self.web_page.setNetworkAccessManager(self.network_manager) self.web_view.setPage(self.web_page) self.web_view.setAttribute(Qt.WA_DeleteOnClose, True) settings = self.web_view.settings() settings.setAttribute(QWebSettings.JavascriptEnabled, True) settings.setAttribute(QWebSettings.PluginsEnabled, False) settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True) settings.setAttribute(QWebSettings.LocalStorageEnabled, True) self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff) self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff) self.deferred = defer.Deferred() request = QNetworkRequest() request.setUrl(QUrl(url)) if baseurl: self._baseUrl = QUrl(baseurl) self.network_manager.finished.connect(self._requestFinished) self.network_manager.get(request) else: self.web_page.loadFinished.connect(self._loadFinished) self.web_page.mainFrame().load(request)
def doRequest(self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, console=False): self.url = url self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time self.js_source = js_source self.js_profile = js_profile self.console = console self.viewport = defaults.VIEWPORT if viewport is None else viewport request = QNetworkRequest() request.setUrl(QUrl(url)) if self.viewport != 'full': # viewport='full' can't be set if content is not loaded yet self._setViewportSize(self.viewport) if baseurl: self._baseUrl = QUrl(baseurl) request.setOriginatingObject(self.web_page.mainFrame()) self._reply = self.network_manager.get(request) self._reply.finished.connect(self._requestFinished) else: self.web_page.loadFinished.connect(self._loadFinished) if self.splash_request.method == 'POST': headers = self.splash_request.getAllHeaders() for header_name, header_value in headers.items(): request.setRawHeader(header_name, header_value) self.web_page.mainFrame().load(request, QNetworkAccessManager.PostOperation, self.splash_request.content.getvalue()) else: self.web_page.mainFrame().load(request)
def checkWeather(self): if self.reply is not None: return request = QNetworkRequest() request.setUrl(QUrl("http://api.liqwei.com/weather/")) self.reply = self.http.get(request) self.reply.finished.connect(self.onReplyFinished) self.reply.error.connect(self.onReplyError)
def doRequest(self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, images=None, console=False): self.url = url self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time self.js_source = js_source self.js_profile = js_profile self.console = console self.viewport = defaults.VIEWPORT if viewport is None else viewport self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, images) # setup logging if self.verbosity >= 4: self.web_page.loadStarted.connect(self._loadStarted) self.web_page.mainFrame().loadFinished.connect(self._frameLoadFinished) self.web_page.mainFrame().loadStarted.connect(self._frameLoadStarted) self.web_page.mainFrame().contentsSizeChanged.connect(self._contentsSizeChanged) if self.verbosity >= 3: self.web_page.mainFrame().javaScriptWindowObjectCleared.connect(self._javaScriptWindowObjectCleared) self.web_page.mainFrame().initialLayoutCompleted.connect(self._initialLayoutCompleted) self.web_page.mainFrame().urlChanged.connect(self._urlChanged) # do the request request = QNetworkRequest() request.setUrl(QUrl(url.decode('utf8'))) if self.viewport != 'full': # viewport='full' can't be set if content is not loaded yet self._setViewportSize(self.viewport) if getattr(self.splash_request, 'pass_headers', False): headers = self.splash_request.getAllHeaders() for name, value in headers.items(): request.setRawHeader(name, value) if name.lower() == 'user-agent': self.web_page.custom_user_agent = value if baseurl: # If baseurl is used, we download the page manually, # then set its contents to the QWebPage and let it # download related resources and render the result. self._baseUrl = QUrl(baseurl.decode('utf8')) request.setOriginatingObject(self.web_page.mainFrame()) self._reply = self.network_manager.get(request) self._reply.finished.connect(self._requestFinished) else: self.web_page.loadFinished.connect(self._loadFinished) if self.splash_request.method == 'POST': body = self.splash_request.content.getvalue() self.web_page.mainFrame().load( request, QNetworkAccessManager.PostOperation, body ) else: self.web_page.mainFrame().load(request)
def prepare_request(self, url_string): url = QUrl(url_string) request = QNetworkRequest() request.setUrl(url) request.setRawHeader('User-Agent', 'Cuivre Web Browser') return request
def DownLoadOverQT (dlURL, LokFileName): # QGIS3: funktioniert unter QGIS 3.x recht zuverlässig auch auf "eingeschränktem Rechner" # - nutzt die Proxyeinstellungen von QGIS # - funktioniert in QGIS selbst auch über HTTPS (es wird durch QGIS eiun Abfragefenster geöffnet) # - bei extrem großen Dateien (z.B. 500MBYte) crasht es bei ReadAll() # QGIS2: funktioniert unter QGIS 2.x innerhalb von QGIS aktuell recht zuverlässig auch auf "eingeschränktem Rechner" # außerhalb hängt sich der Code auf "eingeschräktem Rechner" auf und bringt dann auch kein Ergebnis # Normalrechner funktioniert es def WriteFile(LokFileName, content): # 1. Ziel löschen, wenn existent if path.exists(LokFileName): remove (LokFileName) out=open(LokFileName,'wb') out.write(content) out.close() def onfinish(): WriteFile(LokFileName,reply.readAll()); loop.quit() # 2. Download request = QNetworkRequest() request.setUrl(QUrl(dlURL)) manager = QgsNetworkAccessManager.instance() reply = manager.get(request) reply.setParent(None) loop = QEventLoop() reply.finished.connect(onfinish) loop.exec_() # Wiederholung bei redirekt (13.08.18 Thüringen leitet an HTTPS weiter) status=reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if (status==301): redirectUrl = reply.attribute(request.RedirectionTargetAttribute) request = QNetworkRequest() request.setUrl(redirectUrl) manager = QgsNetworkAccessManager.instance() reply = manager.get(request) reply.setParent(None) loop = QEventLoop() reply.finished.connect(onfinish) loop.exec_() if path.exists(LokFileName): return path.getsize(LokFileName), reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) else: return None, reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) reply.deleteLater()
def request_obj(self, url, headers=None): """ Return a QNetworkRequest object """ request = QNetworkRequest() request.setUrl(QUrl(url)) request.setOriginatingObject(self.web_page.mainFrame()) if headers is not None: self.web_page.skip_custom_headers = True self._set_request_headers(request, headers) return request
def __init__(self, parent=None): # super(MainWindow, self).__init__(parent) super(MainWindow, self).__init__(parent) self.ui = Ui_Dialog() self.ui.setupUi(self) request = QNetworkRequest() request.setUrl(QUrl('http://www.apple.com/cn/retail/shanghaiiapm/')) reply = self.manager.get(request) reply.finished.connect(self.replyFinished) self.ui.webView.load(request)
def doRequest(self, url, baseurl=None, wait_time=None): self.url = url self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time self.deferred = defer.Deferred() request = QNetworkRequest() request.setUrl(QUrl(url)) if baseurl: self._baseUrl = QUrl(baseurl) self.network_manager.finished.connect(self._requestFinished) self.network_manager.get(request) else: self.web_page.loadFinished.connect(self._loadFinished) self.web_page.mainFrame().load(request)
def request_obj(self, url, headers=None, body=None): """ Return a QNetworkRequest object """ request = QNetworkRequest() request.setUrl(to_qurl(url)) request.setOriginatingObject(self.web_page.mainFrame()) if headers is not None: self.web_page.skip_custom_headers = True self._set_request_headers(request, headers) if body and not request.hasRawHeader("content-type"): # there is POST body but no content-type # QT will set this header, but it will complain so better to do this here request.setRawHeader("content-type", "application/x-www-form-urlencoded") return request
def cmdexecute(self,cmd): req = QNetworkRequest() #req = Webkitd.browser.manager.QWebNetworkRequest() req.setUrl(QUrl(Webkitd.url)) req.setRawHeader("Accept-Language","en-us,en;q=0.5"); req.setRawHeader("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7"); req.setRawHeader("Keep-Alive","115"); req.setRawHeader("Connection","keep-alive"); if(Webkitd.browser.referrer != None and Webkitd.browser.referrer != ''): req.setRawHeader("Referer",Webkitd.browser.referrer); Webkitd.browser.load_request(req) if Webkitd.browser._reply_status == False: self.request.send('Error %d %s\n' % (Webkitd.browser._errorCode, Webkitd.browser._errorMessage)) print "%s << Error %d %s" % (self.client_address[0],Webkitd.browser._errorCode, Webkitd.browser._errorMessage) else : self.request.send('ok\n') print "%s << ok" % self.client_address[0]
def __init__(self, url, baseurl=None): QWebPage.__init__(self) self.url = url self.webview = QWebView() self.webview.setPage(self) #self.webview.show() self.deferred = defer.Deferred(self.cancel) if baseurl: self._baseUrl = QUrl(baseurl) request = QNetworkRequest() request.setUrl(QUrl(url)) self.networkAccessManager().finished.connect(self._urlFinished) self.networkAccessManager().get(request) else: self.loadFinished.connect(self._loadFinished) self.mainFrame().load(QUrl(url))
def on_reply_ready(reply, future): nonlocal n_redir # schedule deferred delete to ensure the reply is closed # otherwise we will leak file/socket descriptors reply.deleteLater() future._reply = None if reply.error() == QNetworkReply.OperationCanceledError: # The network request was cancelled reply.close() future.cancel() return if reply.error() != QNetworkReply.NoError: # XXX Maybe convert the error into standard # http and urllib exceptions. future.set_exception(Exception(reply.errorString())) reply.close() return # Handle a possible redirection location = reply.attribute( QNetworkRequest.RedirectionTargetAttribute) if location is not None and n_redir < 1: n_redir += 1 location = reply.url().resolved(location) # Retry the original request with a new url. request = QNetworkRequest(reply.request()) request.setUrl(location) newreply = self._netmanager.get(request) future._reply = newreply newreply.finished.connect( partial(on_reply_ready, newreply, future)) reply.close() return reader = QImageReader(reply) image = reader.read() reply.close() if image.isNull(): future.set_exception(Exception(reader.errorString())) else: future.set_result(image)
def DownLoadOverQT(dlURL, LokFileName): def WriteFile(LokFileName, content): if path.exists(LokFileName): remove(LokFileName) out = open(LokFileName, 'wb') out.write(content) out.close() def onfinish(): WriteFile(LokFileName, reply.readAll()) loop.quit() request = QNetworkRequest() request.setUrl(QUrl(dlURL)) manager = QgsNetworkAccessManager.instance() reply = manager.get(request) reply.setParent(None) loop = QEventLoop() reply.finished.connect(onfinish) loop.exec_() status = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) if (status == 301): redirectUrl = reply.attribute(request.RedirectionTargetAttribute) request = QNetworkRequest() request.setUrl(redirectUrl) manager = QgsNetworkAccessManager.instance() reply = manager.get(request) reply.setParent(None) loop = QEventLoop() reply.finished.connect(onfinish) loop.exec_() if path.exists(LokFileName): return path.getsize(LokFileName), reply.attribute( QNetworkRequest.HttpStatusCodeAttribute) else: return None, reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) reply.deleteLater()
def doRequest(self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, console=False): self.url = url self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time self.js_source = js_source self.js_profile = js_profile self.console = console self.viewport = defaults.VIEWPORT if viewport is None else viewport request = QNetworkRequest() request.setUrl(QUrl(url.decode('utf8'))) if self.viewport != 'full': # viewport='full' can't be set if content is not loaded yet self._setViewportSize(self.viewport) if getattr(self.splash_request, 'pass_headers', False): headers = self.splash_request.getAllHeaders() for name, value in headers.items(): request.setRawHeader(name, value) if name.lower() == 'user-agent': self.web_page.custom_user_agent = value if baseurl: self._baseUrl = QUrl(baseurl.decode('utf8')) request.setOriginatingObject(self.web_page.mainFrame()) self._reply = self.network_manager.get(request) self._reply.finished.connect(self._requestFinished) else: self.web_page.loadFinished.connect(self._loadFinished) if self.splash_request.method == 'POST': self.web_page.mainFrame().load( request, QNetworkAccessManager.PostOperation, self.splash_request.content.getvalue()) else: self.web_page.mainFrame().load(request)
def doRequest( self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, console=False ): self.url = url self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time self.js_source = js_source self.js_profile = js_profile self.console = console self.viewport = defaults.VIEWPORT if viewport is None else viewport request = QNetworkRequest() request.setUrl(QUrl(url.decode("utf8"))) if self.viewport != "full": # viewport='full' can't be set if content is not loaded yet self._setViewportSize(self.viewport) if getattr(self.splash_request, "pass_headers", False): headers = self.splash_request.getAllHeaders() for name, value in headers.items(): request.setRawHeader(name, value) if name.lower() == "user-agent": self.web_page.custom_user_agent = value if baseurl: self._baseUrl = QUrl(baseurl.decode("utf8")) request.setOriginatingObject(self.web_page.mainFrame()) self._reply = self.network_manager.get(request) self._reply.finished.connect(self._requestFinished) else: self.web_page.loadFinished.connect(self._loadFinished) if self.splash_request.method == "POST": self.web_page.mainFrame().load( request, QNetworkAccessManager.PostOperation, self.splash_request.content.getvalue() ) else: self.web_page.mainFrame().load(request)
def on_reply_ready(reply, future): nonlocal n_redir if reply.error() == QNetworkReply.OperationCanceledError: # The network request itself was canceled future.cancel() return if reply.error() != QNetworkReply.NoError: # XXX Maybe convert the error into standard # http and urllib exceptions. future.set_exception(Exception(reply.errorString())) return # Handle a possible redirection location = reply.attribute( QNetworkRequest.RedirectionTargetAttribute) if location is not None and n_redir < 1: n_redir += 1 print(location) location = reply.url().resolved(location) # Retry the original request with a new url. request = QNetworkRequest(reply.request()) request.setUrl(location) newreply = self._netmanager.get(request) future._reply = newreply newreply.finished.connect( partial(on_reply_ready, newreply, future)) return reader = QImageReader(reply) image = reader.read() if image.isNull(): future.set_exception(Exception(reader.errorString())) else: future.set_result(image)
def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True): """ Make a network request by calling QgsNetworkAccessManager. redirections argument is ignored and is here only for httplib2 compatibility. """ self.msg_log(u'http_call request: {0}'.format(url)) self.blocking_mode = blocking req = QNetworkRequest() # Avoid double quoting form QUrl url = urllib.parse.unquote(url) req.setUrl(QUrl(url)) if headers is not None: # This fixes a wierd error with compressed content not being correctly # inflated. # If you set the header on the QNetworkRequest you are basically telling # QNetworkAccessManager "I know what I'm doing, please don't do any content # encoding processing". # See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1 try: del headers['Accept-Encoding'] except KeyError: pass for k, v in list(headers.items()): self.msg_log("Setting header %s to %s" % (k, v)) req.setRawHeader(k, v) # if self.authid: # self.msg_log("Update request w/ authid: {0}".format(self.authid)) # QgsAuthManager.instance().updateNetworkRequest(req, self.authid) if self.reply is not None and self.reply.isRunning(): self.reply.close() if method.lower() == 'delete': func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource') else: func = getattr(QgsNetworkAccessManager.instance(), method.lower()) # Calling the server ... # Let's log the whole call for debugging purposes: self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString())) self.on_abort = False headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()} for k, v in list(headers.items()): self.msg_log("%s: %s" % (k, v)) if method.lower() in ['post', 'put']: if isinstance(body, file): body = body.read() self.reply = func(req, body) else: self.reply = func(req) # if self.authid: # self.msg_log("Update reply w/ authid: {0}".format(self.authid)) # QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid) # necessary to trap local timout manage by QgsNetworkAccessManager # calling QgsNetworkAccessManager::abortRequest QgsNetworkAccessManager.instance().requestTimedOut.connect( self.requestTimedOut) self.reply.sslErrors.connect(self.sslErrors) self.reply.finished.connect(self.replyFinished) self.reply.downloadProgress.connect(self.downloadProgress) # block if blocking mode otherwise return immediatly # it's up to the caller to manage listeners in case of no blocking mode if not self.blocking_mode: return (None, None) # Call and block self.el = QEventLoop() self.reply.finished.connect(self.el.quit) # Catch all exceptions (and clean up requests) try: self.el.exec_(QEventLoop.ExcludeUserInputEvents) except Exception as e: raise e if self.reply: self.reply.finished.disconnect(self.el.quit) # emit exception in case of error if not self.http_call_result.ok: if self.http_call_result.exception and not self.exception_class: raise self.http_call_result.exception else: raise self.exception_class(self.http_call_result.reason) return (self.http_call_result, self.http_call_result.content)
def _create_request(self, url): request = QNetworkRequest() request.setUrl(QUrl(url)) self._set_request_headers(request, self._default_headers) return request
def start(self, url, baseurl=None, wait=None, viewport=None, js_source=None, js_profile=None, images=None, console=False, headers=None, http_method='GET', body=None): self.web_page.har_log.store_timing("_onStarted") self.url = url self.history = [] self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, images) self.wait_time = defaults.WAIT_TIME if wait is None else wait self.js_source = js_source self.js_profile = js_profile self.console = console self.viewport = defaults.VIEWPORT if viewport is None else viewport # setup logging if self.verbosity >= 4: self.web_page.loadStarted.connect(self._loadStarted) self.web_page.mainFrame().loadFinished.connect(self._frameLoadFinished) self.web_page.mainFrame().loadStarted.connect(self._frameLoadStarted) self.web_page.mainFrame().contentsSizeChanged.connect(self._contentsSizeChanged) if self.verbosity >= 3: self.web_page.mainFrame().javaScriptWindowObjectCleared.connect(self._javaScriptWindowObjectCleared) self.web_page.mainFrame().initialLayoutCompleted.connect(self._initialLayoutCompleted) self.web_page.mainFrame().urlChanged.connect(self._urlChanged) # do the request request = QNetworkRequest() request.setUrl(QUrl(url.decode('utf8'))) self._setHeaders(request, headers) if getattr(self.splash_request, 'inspect_me', False): # Set http method and request body from the request http_method = self.splash_request.method body = self.splash_request.content.getvalue() if self.viewport != 'full': # viewport='full' can't be set if content is not loaded yet, # but in other cases it is better to set it earlier. self._setViewportSize(self.viewport) if baseurl: # If baseurl is used, we download the page manually, # then set its contents to the QWebPage and let it # download related resources and render the result. if http_method != 'GET': raise NotImplementedError() self._baseUrl = QUrl(baseurl.decode('utf8')) request.setOriginatingObject(self.web_page.mainFrame()) self._reply = self.network_manager.get(request) self._reply.finished.connect(self._requestFinished) else: self.web_page.loadFinished.connect(self._loadFinished) meth = OPERATION_QT_CONSTANTS[http_method] if body is None: # PyQT doesn't support body=None self.web_page.mainFrame().load(request, meth) else: self.web_page.mainFrame().load(request, meth, body)