def __init__(self, url, baseurl=None):
self.url = url
self.web_view = QWebView()
self.network_manager = SplashQNetworkAccessManager()
self.web_page = SplashQWebPage()
self.web_page.setNetworkAccessManager(self.network_manager)
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
settings = self.web_view.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.deferred = defer.Deferred()
request = QNetworkRequest()
request.setUrl(QUrl(url))
if baseurl:
self._baseUrl = QUrl(baseurl)
self.network_manager.finished.connect(self._requestFinished)
self.network_manager.get(request)
else:
self.web_page.loadFinished.connect(self._loadFinished)
self.web_page.mainFrame().load(request)
def doRequest(self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, console=False):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
request = QNetworkRequest()
request.setUrl(QUrl(url))
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if baseurl:
self._baseUrl = QUrl(baseurl)
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == 'POST':
headers = self.splash_request.getAllHeaders()
for header_name, header_value in headers.items():
request.setRawHeader(header_name, header_value)
self.web_page.mainFrame().load(request,
QNetworkAccessManager.PostOperation,
self.splash_request.content.getvalue())
else:
self.web_page.mainFrame().load(request)
def checkWeather(self):
if self.reply is not None:
return
request = QNetworkRequest()
request.setUrl(QUrl("http://api.liqwei.com/weather/"))
self.reply = self.http.get(request)
self.reply.finished.connect(self.onReplyFinished)
self.reply.error.connect(self.onReplyError)
def checkWeather(self):
if self.reply is not None:
return
request = QNetworkRequest()
request.setUrl(QUrl("http://api.liqwei.com/weather/"))
self.reply = self.http.get(request)
self.reply.finished.connect(self.onReplyFinished)
self.reply.error.connect(self.onReplyError)
def doRequest(self, url, baseurl=None, wait_time=None, viewport=None,
js_source=None, js_profile=None, images=None, console=False):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, images)
# setup logging
if self.verbosity >= 4:
self.web_page.loadStarted.connect(self._loadStarted)
self.web_page.mainFrame().loadFinished.connect(self._frameLoadFinished)
self.web_page.mainFrame().loadStarted.connect(self._frameLoadStarted)
self.web_page.mainFrame().contentsSizeChanged.connect(self._contentsSizeChanged)
if self.verbosity >= 3:
self.web_page.mainFrame().javaScriptWindowObjectCleared.connect(self._javaScriptWindowObjectCleared)
self.web_page.mainFrame().initialLayoutCompleted.connect(self._initialLayoutCompleted)
self.web_page.mainFrame().urlChanged.connect(self._urlChanged)
# do the request
request = QNetworkRequest()
request.setUrl(QUrl(url.decode('utf8')))
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if getattr(self.splash_request, 'pass_headers', False):
headers = self.splash_request.getAllHeaders()
for name, value in headers.items():
request.setRawHeader(name, value)
if name.lower() == 'user-agent':
self.web_page.custom_user_agent = value
if baseurl:
# If baseurl is used, we download the page manually,
# then set its contents to the QWebPage and let it
# download related resources and render the result.
self._baseUrl = QUrl(baseurl.decode('utf8'))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == 'POST':
body = self.splash_request.content.getvalue()
self.web_page.mainFrame().load(
request, QNetworkAccessManager.PostOperation, body
)
else:
self.web_page.mainFrame().load(request)
def prepare_request(self, url_string):
url = QUrl(url_string)
request = QNetworkRequest()
request.setUrl(url)
request.setRawHeader('User-Agent', 'Cuivre Web Browser')
return request
def DownLoadOverQT (dlURL, LokFileName):
# QGIS3: funktioniert unter QGIS 3.x recht zuverlässig auch auf "eingeschränktem Rechner"
# - nutzt die Proxyeinstellungen von QGIS
# - funktioniert in QGIS selbst auch über HTTPS (es wird durch QGIS eiun Abfragefenster geöffnet)
# - bei extrem großen Dateien (z.B. 500MBYte) crasht es bei ReadAll()
# QGIS2: funktioniert unter QGIS 2.x innerhalb von QGIS aktuell recht zuverlässig auch auf "eingeschränktem Rechner"
# außerhalb hängt sich der Code auf "eingeschräktem Rechner" auf und bringt dann auch kein Ergebnis
# Normalrechner funktioniert es
def WriteFile(LokFileName, content):
# 1. Ziel löschen, wenn existent
if path.exists(LokFileName):
remove (LokFileName)
out=open(LokFileName,'wb')
out.write(content)
out.close()
def onfinish():
WriteFile(LokFileName,reply.readAll());
loop.quit()
# 2. Download
request = QNetworkRequest()
request.setUrl(QUrl(dlURL))
manager = QgsNetworkAccessManager.instance()
reply = manager.get(request)
reply.setParent(None)
loop = QEventLoop()
reply.finished.connect(onfinish)
loop.exec_()
# Wiederholung bei redirekt (13.08.18 Thüringen leitet an HTTPS weiter)
status=reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if (status==301):
redirectUrl = reply.attribute(request.RedirectionTargetAttribute)
request = QNetworkRequest()
request.setUrl(redirectUrl)
manager = QgsNetworkAccessManager.instance()
reply = manager.get(request)
reply.setParent(None)
loop = QEventLoop()
reply.finished.connect(onfinish)
loop.exec_()
if path.exists(LokFileName):
return path.getsize(LokFileName), reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
else:
return None, reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
reply.deleteLater()
def request_obj(self, url, headers=None):
""" Return a QNetworkRequest object """
request = QNetworkRequest()
request.setUrl(QUrl(url))
request.setOriginatingObject(self.web_page.mainFrame())
if headers is not None:
self.web_page.skip_custom_headers = True
self._set_request_headers(request, headers)
return request
def __init__(self, parent=None):
# super(MainWindow, self).__init__(parent)
super(MainWindow, self).__init__(parent)
self.ui = Ui_Dialog()
self.ui.setupUi(self)
request = QNetworkRequest()
request.setUrl(QUrl('http://www.apple.com/cn/retail/shanghaiiapm/'))
reply = self.manager.get(request)
reply.finished.connect(self.replyFinished)
self.ui.webView.load(request)
def request_obj(self, url, headers=None):
""" Return a QNetworkRequest object """
request = QNetworkRequest()
request.setUrl(QUrl(url))
request.setOriginatingObject(self.web_page.mainFrame())
if headers is not None:
self.web_page.skip_custom_headers = True
self._set_request_headers(request, headers)
return request
def __init__(self, parent=None):
# super(MainWindow, self).__init__(parent)
super(MainWindow, self).__init__(parent)
self.ui = Ui_Dialog()
self.ui.setupUi(self)
request = QNetworkRequest()
request.setUrl(QUrl('http://www.apple.com/cn/retail/shanghaiiapm/'))
reply = self.manager.get(request)
reply.finished.connect(self.replyFinished)
self.ui.webView.load(request)
def doRequest(self, url, baseurl=None, wait_time=None):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.deferred = defer.Deferred()
request = QNetworkRequest()
request.setUrl(QUrl(url))
if baseurl:
self._baseUrl = QUrl(baseurl)
self.network_manager.finished.connect(self._requestFinished)
self.network_manager.get(request)
else:
self.web_page.loadFinished.connect(self._loadFinished)
self.web_page.mainFrame().load(request)
def request_obj(self, url, headers=None, body=None):
""" Return a QNetworkRequest object """
request = QNetworkRequest()
request.setUrl(to_qurl(url))
request.setOriginatingObject(self.web_page.mainFrame())
if headers is not None:
self.web_page.skip_custom_headers = True
self._set_request_headers(request, headers)
if body and not request.hasRawHeader("content-type"):
# there is POST body but no content-type
# QT will set this header, but it will complain so better to do this here
request.setRawHeader("content-type", "application/x-www-form-urlencoded")
return request
def request_obj(self, url, headers=None, body=None):
""" Return a QNetworkRequest object """
request = QNetworkRequest()
request.setUrl(to_qurl(url))
request.setOriginatingObject(self.web_page.mainFrame())
if headers is not None:
self.web_page.skip_custom_headers = True
self._set_request_headers(request, headers)
if body and not request.hasRawHeader("content-type"):
# there is POST body but no content-type
# QT will set this header, but it will complain so better to do this here
request.setRawHeader("content-type",
"application/x-www-form-urlencoded")
return request
def cmdexecute(self,cmd):
req = QNetworkRequest()
#req = Webkitd.browser.manager.QWebNetworkRequest()
req.setUrl(QUrl(Webkitd.url))
req.setRawHeader("Accept-Language","en-us,en;q=0.5");
req.setRawHeader("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
req.setRawHeader("Keep-Alive","115");
req.setRawHeader("Connection","keep-alive");
if(Webkitd.browser.referrer != None and Webkitd.browser.referrer != ''):
req.setRawHeader("Referer",Webkitd.browser.referrer);
Webkitd.browser.load_request(req)
if Webkitd.browser._reply_status == False:
self.request.send('Error %d %s\n' % (Webkitd.browser._errorCode, Webkitd.browser._errorMessage))
print "%s << Error %d %s" % (self.client_address[0],Webkitd.browser._errorCode, Webkitd.browser._errorMessage)
else :
self.request.send('ok\n')
print "%s << ok" % self.client_address[0]
def __init__(self, url, baseurl=None):
QWebPage.__init__(self)
self.url = url
self.webview = QWebView()
self.webview.setPage(self)
#self.webview.show()
self.deferred = defer.Deferred(self.cancel)
if baseurl:
self._baseUrl = QUrl(baseurl)
request = QNetworkRequest()
request.setUrl(QUrl(url))
self.networkAccessManager().finished.connect(self._urlFinished)
self.networkAccessManager().get(request)
else:
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
def on_reply_ready(reply, future):
nonlocal n_redir
# schedule deferred delete to ensure the reply is closed
# otherwise we will leak file/socket descriptors
reply.deleteLater()
future._reply = None
if reply.error() == QNetworkReply.OperationCanceledError:
# The network request was cancelled
reply.close()
future.cancel()
return
if reply.error() != QNetworkReply.NoError:
# XXX Maybe convert the error into standard
# http and urllib exceptions.
future.set_exception(Exception(reply.errorString()))
reply.close()
return
# Handle a possible redirection
location = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if location is not None and n_redir < 1:
n_redir += 1
location = reply.url().resolved(location)
# Retry the original request with a new url.
request = QNetworkRequest(reply.request())
request.setUrl(location)
newreply = self._netmanager.get(request)
future._reply = newreply
newreply.finished.connect(
partial(on_reply_ready, newreply, future))
reply.close()
return
reader = QImageReader(reply)
image = reader.read()
reply.close()
if image.isNull():
future.set_exception(Exception(reader.errorString()))
else:
future.set_result(image)
def DownLoadOverQT(dlURL, LokFileName):
def WriteFile(LokFileName, content):
if path.exists(LokFileName):
remove(LokFileName)
out = open(LokFileName, 'wb')
out.write(content)
out.close()
def onfinish():
WriteFile(LokFileName, reply.readAll())
loop.quit()
request = QNetworkRequest()
request.setUrl(QUrl(dlURL))
manager = QgsNetworkAccessManager.instance()
reply = manager.get(request)
reply.setParent(None)
loop = QEventLoop()
reply.finished.connect(onfinish)
loop.exec_()
status = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if (status == 301):
redirectUrl = reply.attribute(request.RedirectionTargetAttribute)
request = QNetworkRequest()
request.setUrl(redirectUrl)
manager = QgsNetworkAccessManager.instance()
reply = manager.get(request)
reply.setParent(None)
loop = QEventLoop()
reply.finished.connect(onfinish)
loop.exec_()
if path.exists(LokFileName):
return path.getsize(LokFileName), reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
else:
return None, reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
reply.deleteLater()
def doRequest(self,
url,
baseurl=None,
wait_time=None,
viewport=None,
js_source=None,
js_profile=None,
console=False):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
request = QNetworkRequest()
request.setUrl(QUrl(url.decode('utf8')))
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if getattr(self.splash_request, 'pass_headers', False):
headers = self.splash_request.getAllHeaders()
for name, value in headers.items():
request.setRawHeader(name, value)
if name.lower() == 'user-agent':
self.web_page.custom_user_agent = value
if baseurl:
self._baseUrl = QUrl(baseurl.decode('utf8'))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == 'POST':
self.web_page.mainFrame().load(
request, QNetworkAccessManager.PostOperation,
self.splash_request.content.getvalue())
else:
self.web_page.mainFrame().load(request)
def doRequest(
self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, console=False
):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
request = QNetworkRequest()
request.setUrl(QUrl(url.decode("utf8")))
if self.viewport != "full":
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if getattr(self.splash_request, "pass_headers", False):
headers = self.splash_request.getAllHeaders()
for name, value in headers.items():
request.setRawHeader(name, value)
if name.lower() == "user-agent":
self.web_page.custom_user_agent = value
if baseurl:
self._baseUrl = QUrl(baseurl.decode("utf8"))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == "POST":
self.web_page.mainFrame().load(
request, QNetworkAccessManager.PostOperation, self.splash_request.content.getvalue()
)
else:
self.web_page.mainFrame().load(request)
def on_reply_ready(reply, future):
nonlocal n_redir
if reply.error() == QNetworkReply.OperationCanceledError:
# The network request itself was canceled
future.cancel()
return
if reply.error() != QNetworkReply.NoError:
# XXX Maybe convert the error into standard
# http and urllib exceptions.
future.set_exception(Exception(reply.errorString()))
return
# Handle a possible redirection
location = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if location is not None and n_redir < 1:
n_redir += 1
print(location)
location = reply.url().resolved(location)
# Retry the original request with a new url.
request = QNetworkRequest(reply.request())
request.setUrl(location)
newreply = self._netmanager.get(request)
future._reply = newreply
newreply.finished.connect(
partial(on_reply_ready, newreply, future))
return
reader = QImageReader(reply)
image = reader.read()
if image.isNull():
future.set_exception(Exception(reader.errorString()))
else:
future.set_result(image)
def on_reply_ready(reply, future):
nonlocal n_redir
if reply.error() == QNetworkReply.OperationCanceledError:
# The network request itself was canceled
future.cancel()
return
if reply.error() != QNetworkReply.NoError:
# XXX Maybe convert the error into standard
# http and urllib exceptions.
future.set_exception(Exception(reply.errorString()))
return
# Handle a possible redirection
location = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if location is not None and n_redir < 1:
n_redir += 1
print(location)
location = reply.url().resolved(location)
# Retry the original request with a new url.
request = QNetworkRequest(reply.request())
request.setUrl(location)
newreply = self._netmanager.get(request)
future._reply = newreply
newreply.finished.connect(
partial(on_reply_ready, newreply, future))
return
reader = QImageReader(reply)
image = reader.read()
if image.isNull():
future.set_exception(Exception(reader.errorString()))
else:
future.set_result(image)
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None,
blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k, v)
# if self.authid:
# self.msg_log("Update request w/ authid: {0}".format(self.authid))
# QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, file):
body = body.read()
self.reply = func(req, body)
else:
self.reply = func(req)
# if self.authid:
# self.msg_log("Update reply w/ authid: {0}".format(self.authid))
# QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)
# necessary to trap local timout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(
self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
def _create_request(self, url):
request = QNetworkRequest()
request.setUrl(QUrl(url))
self._set_request_headers(request, self._default_headers)
return request
def _create_request(self, url):
request = QNetworkRequest()
request.setUrl(QUrl(url))
self._set_request_headers(request, self._default_headers)
return request
def start(self, url, baseurl=None, wait=None, viewport=None,
js_source=None, js_profile=None, images=None, console=False,
headers=None, http_method='GET', body=None):
self.web_page.har_log.store_timing("_onStarted")
self.url = url
self.history = []
self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, images)
self.wait_time = defaults.WAIT_TIME if wait is None else wait
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
# setup logging
if self.verbosity >= 4:
self.web_page.loadStarted.connect(self._loadStarted)
self.web_page.mainFrame().loadFinished.connect(self._frameLoadFinished)
self.web_page.mainFrame().loadStarted.connect(self._frameLoadStarted)
self.web_page.mainFrame().contentsSizeChanged.connect(self._contentsSizeChanged)
if self.verbosity >= 3:
self.web_page.mainFrame().javaScriptWindowObjectCleared.connect(self._javaScriptWindowObjectCleared)
self.web_page.mainFrame().initialLayoutCompleted.connect(self._initialLayoutCompleted)
self.web_page.mainFrame().urlChanged.connect(self._urlChanged)
# do the request
request = QNetworkRequest()
request.setUrl(QUrl(url.decode('utf8')))
self._setHeaders(request, headers)
if getattr(self.splash_request, 'inspect_me', False):
# Set http method and request body from the request
http_method = self.splash_request.method
body = self.splash_request.content.getvalue()
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet,
# but in other cases it is better to set it earlier.
self._setViewportSize(self.viewport)
if baseurl:
# If baseurl is used, we download the page manually,
# then set its contents to the QWebPage and let it
# download related resources and render the result.
if http_method != 'GET':
raise NotImplementedError()
self._baseUrl = QUrl(baseurl.decode('utf8'))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
meth = OPERATION_QT_CONSTANTS[http_method]
if body is None: # PyQT doesn't support body=None
self.web_page.mainFrame().load(request, meth)
else:
self.web_page.mainFrame().load(request, meth, body)
