def open(self, address, method='get', headers={}, auth=None, body=None,
default_popup_response=None):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tuple of HTTP auth (username, password).
:param body: An optional string containing a payload.
:param default_popup_response: the default response for any confirm/
alert/prompt popup from the Javascript (replaces the need for the with
blocks)
:return: Page resource, All loaded resources.
"""
body = body or QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Error("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl(0)
for header in headers:
request.setRawHeader(header, headers[header])
self._auth = auth
self._auth_attempt = 0 # Avoids reccursion
self.main_frame.load(request, method, body)
self.loaded = False
Ghost._prompt_expected = (default_popup_response, None)
Ghost._confirm_expected = (default_popup_response, None)
return self.wait_for_page_loaded()
def request(self, iri, method="GET", extra_headers=None,
oauth_credentials=None, post_content=None):
"""Return a deferred that will be fired with a Response object."""
uri = self.iri_to_uri(iri)
request = QNetworkRequest(QUrl(uri))
request.setSslConfiguration(self.ssl_config)
headers = yield self.build_request_headers(uri, method, extra_headers,
oauth_credentials)
for key, value in headers.items():
request.setRawHeader(key, value)
post_buffer = QBuffer()
post_buffer.setData(post_content)
try:
result = yield self._perform_request(request, method, post_buffer)
except ProxyUnauthorizedError as e:
app_proxy = QNetworkProxy.applicationProxy()
proxy_host = app_proxy.hostName() if app_proxy else "proxy server"
got_creds = yield self.request_proxy_auth_credentials(
proxy_host, self.proxy_retry)
if got_creds:
self.proxy_retry = True
result = yield self.request(iri, method, extra_headers,
oauth_credentials, post_content)
else:
excp = WebClientError('Proxy creds needed.', e)
defer.returnValue(excp)
defer.returnValue(result)
def open(self, address, method='get', headers={}, auth=None):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tupple of HTTP auth (Basic, username, password).
:return: Page resource, All loaded resources.
"""
body = QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
if auth is not None:
auth_type, username, password = auth
def authenticate(reply, authenticator):
authenticator.setUser(username)
authenticator.setPassword(password)
self.page.networkAccessManager().authenticationRequired.connect(authenticate)
if not "User-Agent" in headers:
headers["User-Agent"] = self.user_agent
for header in headers:
request.setRawHeader(header, headers[header])
self.main_frame.load(request, method, body)
self.loaded = False
return self.wait_for_page_loaded()
def open(self, address, method='get', headers={}, auth=None, body=None):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tupple of HTTP auth (username, password).
:param body: An optional string containing a payload.
:return: Page resource, All loaded resources.
"""
body = body or QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl(0)
for header in headers:
request.setRawHeader(header, headers[header])
self._auth = auth
self._auth_attempt = 0 # Avoids reccursion
self.main_frame.load(request, method, body)
self.loaded = False
return self.wait_for_page_loaded()
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('q', query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def make_request(self, url):
url = QUrl(url)
req = QNetworkRequest(url)
for header in self.headers:
val = self.headers[header]
req.setRawHeader(header, val)
return req
def createRequest(self, operation, request, outgoingData=None):
request = QNetworkRequest(request)
old_proxy = self.proxy()
# Hack to change Language
request.setRawHeader("Accept-Language","fr-FR,fr;q=0.8");
#request.setRawHeader("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
#request.setRawHeader("Connection","keep-alive");
splash_proxy_factory = self._getSplashProxyFactory(request)
if splash_proxy_factory:
proxy_query = QNetworkProxyQuery(request.url())
proxy = splash_proxy_factory.queryProxy(proxy_query)[0]
self.setProxy(proxy)
# this method is called createRequest, but in fact it creates a reply
reply = super(ProxiedQNetworkAccessManager, self).createRequest(
operation, request, outgoingData
)
reply.error.connect(self._handleError)
reply.finished.connect(self._handleFinished)
reply.metaDataChanged.connect(self._handleMetaData)
reply.downloadProgress.connect(self._handleDownloadProgress)
self.setProxy(old_proxy)
return reply
def __page_load_slot(self):
with self.__page_load_condition:
request = self.__page_load_request
# setea la operación
if request.method == 'GET':
operation = QNetworkAccessManager.GetOperation
elif request.method == 'POST':
operation = QNetworkAccessManager.PostOperation
else:
raise ValueError("Unsupported method %s" % request.method)
user_agent = None # el user agent para el request
# Si se definieron headers personalizados se setean tambien
qnetworkrequest =QNetworkRequest(UrlHelper.from_user_input(request.url))
for header, value in list(self.__custom_headers.items()):
qnetworkrequest.setRawHeader(header, value)
if header.lower() == 'user-agent':
user_agent = value
# setea url y headers
qnetworkrequest =QNetworkRequest(UrlHelper.from_user_input(request.url))
for header, value in list(request.headers.items()):
qnetworkrequest.setRawHeader(header, value)
if header.lower() == 'user-agent':
user_agent = value
self.__page._set_user_agent(user_agent)
# lo más importante: carga la página
self.__web_view.load(qnetworkrequest, operation, request.body)
def read(self, **kwargs):
self.connect(self.manager, SIGNAL("finished( QNetworkReply* )"), self.readRequestFinished)
params = []
try:
queryable = kwargs["queryable"]
params.append({"queryable": queryable})
queryableList = QString(queryable).split(",")
except KeyError:
pass
for key, value in kwargs.items():
if QString(key).split("__")[0] in queryableList:
params.append({key: value})
url = "%s/stakeholders?" % self.host
for p in params:
url = "%s%s=%s&" % (url, p.keys()[0], p.values()[0])
qUrl = QUrl(url)
# Create a new request
request = QNetworkRequest(qUrl)
request.setRawHeader("Authorization", self.login)
request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json")
self.manager.get(request)
return url
def doRequest(self, url, baseurl=None, wait_time=None, viewport=None, js_source=None, js_profile=None, console=False):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
request = QNetworkRequest()
request.setUrl(QUrl(url))
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if baseurl:
self._baseUrl = QUrl(baseurl)
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == 'POST':
headers = self.splash_request.getAllHeaders()
for header_name, header_value in headers.items():
request.setRawHeader(header_name, header_value)
self.web_page.mainFrame().load(request,
QNetworkAccessManager.PostOperation,
self.splash_request.content.getvalue())
else:
self.web_page.mainFrame().load(request)
def createRequest(self, op, request, outgoingData=None):
"""
Protected method to create a request.
@param op the operation to be performed (QNetworkAccessManager.Operation)
@param request reference to the request object (QNetworkRequest)
@param outgoingData reference to an IODevice containing data to be sent
(QIODevice)
@return reference to the created reply object (QNetworkReply)
"""
scheme = request.url().scheme()
if scheme == "https" and (not SSL_AVAILABLE or not QSslSocket.supportsSsl()):
return NetworkProtocolUnknownErrorReply(scheme, self)
if op == QNetworkAccessManager.PostOperation and outgoingData is not None:
outgoingDataByteArray = outgoingData.peek(1024 * 1024)
Helpviewer.HelpWindow.HelpWindow.passwordManager().post(request, outgoingDataByteArray)
reply = None
if unicode(scheme) in self.__schemeHandlers:
reply = self.__schemeHandlers[unicode(scheme)].createRequest(op, request, outgoingData)
if reply is not None:
return reply
if not self.__acceptLanguage.isEmpty():
req = QNetworkRequest(request)
req.setRawHeader("Accept-Language", self.__acceptLanguage)
else:
req = request
# AdBlock code
if op == QNetworkAccessManager.GetOperation:
if self.__adblockNetwork is None:
self.__adblockNetwork = Helpviewer.HelpWindow.HelpWindow.adblockManager().network()
reply = self.__adblockNetwork.block(req)
if reply is not None:
reply.setParent(self)
return reply
# set cache policy
if op == QNetworkAccessManager.GetOperation:
urlHost = req.url().host()
for host in self.NoCacheHosts:
if host in urlHost:
req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork)
break
else:
req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork)
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
self.emit(
SIGNAL("requestCreated(QNetworkAccessManager::Operation, const QNetworkRequest&, QNetworkReply*)"),
op,
req,
reply,
)
return reply
def prepare_request(self, url_string):
url = QUrl(url_string)
request = QNetworkRequest()
request.setUrl(url)
request.setRawHeader('User-Agent', 'Cuivre Web Browser')
return request
def doRequest(self, url, baseurl=None, wait_time=None, viewport=None,
js_source=None, js_profile=None, images=None, console=False):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, images)
# setup logging
if self.verbosity >= 4:
self.web_page.loadStarted.connect(self._loadStarted)
self.web_page.mainFrame().loadFinished.connect(self._frameLoadFinished)
self.web_page.mainFrame().loadStarted.connect(self._frameLoadStarted)
self.web_page.mainFrame().contentsSizeChanged.connect(self._contentsSizeChanged)
if self.verbosity >= 3:
self.web_page.mainFrame().javaScriptWindowObjectCleared.connect(self._javaScriptWindowObjectCleared)
self.web_page.mainFrame().initialLayoutCompleted.connect(self._initialLayoutCompleted)
self.web_page.mainFrame().urlChanged.connect(self._urlChanged)
# do the request
request = QNetworkRequest()
request.setUrl(QUrl(url.decode('utf8')))
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if getattr(self.splash_request, 'pass_headers', False):
headers = self.splash_request.getAllHeaders()
for name, value in headers.items():
request.setRawHeader(name, value)
if name.lower() == 'user-agent':
self.web_page.custom_user_agent = value
if baseurl:
# If baseurl is used, we download the page manually,
# then set its contents to the QWebPage and let it
# download related resources and render the result.
self._baseUrl = QUrl(baseurl.decode('utf8'))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == 'POST':
body = self.splash_request.content.getvalue()
self.web_page.mainFrame().load(
request, QNetworkAccessManager.PostOperation, body
)
else:
self.web_page.mainFrame().load(request)
def get(self, url):
future = Future()
url = url = QUrl(url)
request = QNetworkRequest(url)
request.setRawHeader("User-Agent", "OWImageViewer/1.0")
request.setAttribute(
QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache
)
# Future yielding a QNetworkReply when finished.
reply = self._netmanager.get(request)
future._reply = reply
n_redir = 0
def on_reply_ready(reply, future):
nonlocal n_redir
if reply.error() == QNetworkReply.OperationCanceledError:
# The network request itself was canceled
future.cancel()
return
if reply.error() != QNetworkReply.NoError:
# XXX Maybe convert the error into standard
# http and urllib exceptions.
future.set_exception(Exception(reply.errorString()))
return
# Handle a possible redirection
location = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if location is not None and n_redir < 1:
n_redir += 1
print(location)
location = reply.url().resolved(location)
# Retry the original request with a new url.
request = QNetworkRequest(reply.request())
request.setUrl(location)
newreply = self._netmanager.get(request)
future._reply = newreply
newreply.finished.connect(
partial(on_reply_ready, newreply, future))
return
reader = QImageReader(reply)
image = reader.read()
if image.isNull():
future.set_exception(Exception(reader.errorString()))
else:
future.set_result(image)
reply.finished.connect(partial(on_reply_ready, reply, future))
return future
def sendVote(self, plugin_id, vote):
""" send vote via the RPC """
if not plugin_id or not vote:
return False
url = "http://plugins.qgis.org/plugins/RPC2/"
params = "{\"id\":\"djangorpc\",\"method\":\"plugin.vote\",\"params\":[%s,%s]}" % (str(plugin_id), str(vote))
req = QNetworkRequest(QUrl(url))
req.setRawHeader("Content-Type", "application/json")
QgsNetworkAccessManager.instance().post(req, params)
return True
def executeRequest (self, url, callback, post = None):
self.messageBar.clearWidgets()
self.cmbOfferings.setEnabled (False)
self.tabWidget.setEnabled (False)
if post:
request = QNetworkRequest(url)
request.setRawHeader('Content-Type', 'application/xml')
self.reply = QgsNetworkAccessManager.instance().post(request,post)
else:
self.reply = QgsNetworkAccessManager.instance().get(QNetworkRequest(url))
progressMessageBar = self.messageBar.createMessage(self.tr("Please wait while downloading"))
progressBar = QtGui.QProgressBar(self)
progressBar.setMinimum(0)
progressBar.setFormat(self.tr("%v bytes downloaded!"))
progressBar.setAlignment(QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
def updateProgress (read, total):
try:
progressBar.setMaximum (total)
progressBar.setValue (read)
except: pass
updateProgress (0,0)
self.reply.downloadProgress.connect (updateProgress)
progressMessageBar.layout().addWidget(progressBar)
btnAbort = QtGui.QPushButton(self.tr("Abort"))
btnAbort.clicked.connect(self.reply.abort)
progressMessageBar.layout().addWidget(btnAbort)
fd, replyFilename = mkstemp(suffix=".xml", prefix = callback.__name__, dir = self.selectedWorkDir())
os.close(fd)
replyFile = open (replyFilename, "w")
def replyReadyRead ():
try: replyFile.write (self.reply.readAll())
except: pass
self.reply.readyRead.connect(replyReadyRead)
def finishRequest ():
replyFile.close()
self.cmbOfferings.setEnabled (True)
self.tabWidget.setEnabled(True)
self.messageBar.clearWidgets()
if self.reply.error() != QNetworkReply.NoError:
self.messageBar.pushMessage(self.reply.errorString(), QgsMessageBar.CRITICAL)
else:
callback(replyFilename)
self.reply.deleteLater()
self.reply.finished.connect(finishRequest)
self.messageBar.pushWidget(progressMessageBar, QgsMessageBar.INFO)
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(
r'output="[a-z]*"', 'output="' + self.__output + '"', query)
query = re.sub(
r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
encoded_query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('data', encoded_query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, self.data):
raise OverpassTimeoutException
else:
return self.data
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass API")
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
request = QNetworkRequest(QUrl(url))
request.setRawHeader("User-Agent", self.userAgent)
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self.replyFinished)
self.requestingUrls.append(url)
self.replies.append(reply)
return reply
class SettingsProtocol(QObject):
readSignal = pyqtSignal(bool, int, QString)
def __init__(self, host, user, password):
QObject.__init__(self)
# Set the host
self.host = host
# Create a base64 encoded credential string from user name and password.
# This is required for the HTTP basic access authentication, compare
# also http://en.wikipedia.org/wiki/Basic_access_authentication
self.userlogin = "******" % (user, password)
self.login = "******" + QByteArray(self.userlogin).toBase64()
# Create a new QNetworkAccessManager and connect it to the
# authenticationRequired signal
self.manager = QNetworkAccessManager(self)
# self.connect(self.manager, SIGNAL("authenticationRequired( QNetworkReply*, QAuthenticator* )"), self.slotAuthenticationRequired)
def read(self):
self.connect(self.manager, SIGNAL("finished( QNetworkReply* )"), self.readRequestFinished)
url = "%s/config/geometrytaggroups" % self.host
qUrl = QUrl(url)
self.request = QNetworkRequest(qUrl)
self.request.setRawHeader("Authorization", self.login)
self.request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json")
self.manager.get(self.request)
return url
def readRequestFinished(self, reply):
# Get the HTTP status code from the reply
self.httpStatusCode = int(reply.attribute(QNetworkRequest.HttpStatusCodeAttribute).toString())
data = reply.readAll()
# Check the status code see also http://en.wikipedia.org/wiki/HTTP_status_code
# In case of a successful upload we get a 201 status code back and the
# "stylePosted" signal is emitted with the first parameter set to True.
# If the query didn't succeed, the status code is 4xx indicating that
# the host was not found, the authentication failed or a forbidden
# action in case a style with the same name already exists. It's up to
# the receiver to handle these status codes.
self.readSignal.emit(self.httpStatusCode in (200, 201), self.httpStatusCode, QString(data))
def request_obj(self, url, headers=None, body=None):
""" Return a QNetworkRequest object """
request = QNetworkRequest()
request.setUrl(to_qurl(url))
request.setOriginatingObject(self.web_page.mainFrame())
if headers is not None:
self.web_page.skip_custom_headers = True
self._set_request_headers(request, headers)
if body and not request.hasRawHeader("content-type"):
# there is POST body but no content-type
# QT will set this header, but it will complain so better to do this here
request.setRawHeader("content-type", "application/x-www-form-urlencoded")
return request
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
# create request
request = QNetworkRequest(QUrl(url))
if self.userAgent:
request.setRawHeader("User-Agent", self.userAgent) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2
# send request
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self._replyFinished)
self.requestingReplies[url] = reply
return reply
def upload(self, filePath, returnDict=True):
self.returnDict = returnDict
file = QFile(filePath)
file.open(QFile.ReadOnly)
url = QUrl(self.apiUrl + "imports/?api_key={}".format(self.apiKey))
files = {"file": file}
multipart = self._createMultipart(files=files)
request = QNetworkRequest(url)
request.setHeader(QNetworkRequest.ContentTypeHeader, "multipart/form-data; boundary=%s" % multipart.boundary())
request.setRawHeader("User-Agent", "QGISCartoDB 0.2.x")
reply = self.manager.post(request, multipart)
loop = QEventLoop()
reply.uploadProgress.connect(self.progressCB)
reply.error.connect(self._error)
reply.finished.connect(loop.exit)
loop.exec_()
def upload(self, filePath, returnDict=True):
self.returnDict = returnDict
file = QFile(filePath)
file.open(QFile.ReadOnly)
apiUrl = 'https://{}.cartodb.com/api/v1/imports/?api_key={}'.format(self.cartodbUser, self.apiKey)
url = QUrl(apiUrl)
files = {'file': file}
multipart = self._createMultipart(files=files)
request = QNetworkRequest(url)
request.setHeader(QNetworkRequest.ContentTypeHeader, 'multipart/form-data; boundary=%s' % multipart.boundary())
request.setRawHeader('User-Agent', 'QGIS 2.x')
reply = self.manager.post(request, multipart)
loop = QEventLoop()
reply.uploadProgress.connect(self.progressCB)
reply.error.connect(self.error)
reply.finished.connect(loop.exit)
loop.exec_()
def cmdexecute(self,cmd):
req = QNetworkRequest()
#req = Webkitd.browser.manager.QWebNetworkRequest()
req.setUrl(QUrl(Webkitd.url))
req.setRawHeader("Accept-Language","en-us,en;q=0.5");
req.setRawHeader("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
req.setRawHeader("Keep-Alive","115");
req.setRawHeader("Connection","keep-alive");
if(Webkitd.browser.referrer != None and Webkitd.browser.referrer != ''):
req.setRawHeader("Referer",Webkitd.browser.referrer);
Webkitd.browser.load_request(req)
if Webkitd.browser._reply_status == False:
self.request.send('Error %d %s\n' % (Webkitd.browser._errorCode, Webkitd.browser._errorMessage))
print "%s << Error %d %s" % (self.client_address[0],Webkitd.browser._errorCode, Webkitd.browser._errorMessage)
else :
self.request.send('ok\n')
print "%s << ok" % self.client_address[0]
def load(self, url, headers={}, body="", load_timeout=-1, delay=None):
_headers = self.headers.copy()
_headers.update(headers)
self._load_success = False
self._load_timeout = load_timeout
url = QUrl(url)
req = QNetworkRequest(url)
for header in _headers:
val = _headers[header]
req.setRawHeader(header, val)
self.webframe.load(req, QNetworkAccessManager.GetOperation, body)
# wait to load finished
self._wait_finish()
# delay wait to render html
if delay:
self.wait_delays(delay)
def ask_for_token(c_id, c_secret, request_status=True):
"""Ask a token from Isogeo API authentification page.
This send a POST request to Isogeo API with the user id and secret in
its header. The API should return an access token
"""
headervalue = "Basic " + base64.b64encode(c_id + ":" + c_secret)
data = urllib.urlencode({"grant_type": "client_credentials"})
databyte = QByteArray()
databyte.append(data)
url = QUrl('https://id.api.isogeo.com/oauth/token')
request = QNetworkRequest(url)
request.setRawHeader("Authorization", headervalue)
if request_status is True:
request_status = False
return request, databyte
QgsMessageLog.logMessage("Authentication succeeded", "Isogeo")
def open(self, address, method='get'):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:return: Page resource, All loaded resources.
"""
body = QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.setRawHeader("User-Agent", self.user_agent)
self.main_frame.load(request, method, body)
self.loaded = False
return self.wait_for_page_loaded()
def _request(self, path, feed = False, post_args = None, **url_args):
url = self.url.resolved(QUrl(path))
for key in url_args:
url.addQueryItem(key, url_args[key])
url.addQueryItem("format", "json")
print "ff req:", url
req = QNetworkRequest(url)
if self.session and self.session.is_auth():
req.setRawHeader("Authorization", "Basic " + base64.b64encode("%s:%s" % (self.session.user, self.session.remoteKey)))
if post_args:
data = urllib.urlencode(post_args)
reply = self.manager.post(req, data)
else:
reply = self.manager.get(req)
req = FriendFeedRequest(reply, feed)
self.activeRequests.add(req)
self.connect(req, SIGNAL("cleanup"), self._cleanup)
return req
def load(self, url, headers={}, body="", load_timeout=-1, delay=None):
_headers = self.headers.copy()
_headers.update(headers)
self._load_success = False
self._load_timeout = load_timeout
url = QUrl(url)
req = QNetworkRequest(url)
for header in _headers:
val = _headers[header]
req.setRawHeader(header, val)
self.webframe.load(req, QNetworkAccessManager.GetOperation, body)
# wait to load finished
self._wait_finish()
# delay wait to render html
if delay:
self.wait_delays(delay)
def upload(self, filePath, returnDict=True):
self.returnDict = returnDict
file = QFile(filePath)
file.open(QFile.ReadOnly)
url = QUrl(self.apiUrl + "imports/?api_key={}".format(self.apiKey))
files = {'file': file}
multipart = self._createMultipart(files=files)
request = QNetworkRequest(url)
request.setHeader(
QNetworkRequest.ContentTypeHeader,
'multipart/form-data; boundary=%s' % multipart.boundary())
request.setRawHeader('User-Agent', 'QGISCartoDB 0.2.x')
reply = self.manager.post(request, multipart)
loop = QEventLoop()
reply.uploadProgress.connect(self.progressCB)
reply.error.connect(self._error)
reply.finished.connect(loop.exit)
loop.exec_()
def open(self, address, method='get', headers={}, auth=None,
wait_onload_event=True, wait_for_loading=True):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tupple of HTTP auth (username, password).
:param wait_onload_event: If it's set to True waits until the OnLoad event from
the main page is fired. Otherwise wait until the Dom is ready.
:param wait_for_loading: If True waits until the page is Loaded. Note that wait_onload_event
isn't valid if wait_for_loading is False.
:return: Page resource, All loaded resources.
"""
if not wait_onload_event:
if self.ghostInit.receivers(SIGNAL("dom_is_ready(bool)")) == 0:
self.ghostInit.dom_is_ready.connect(self._page_loaded)
Logger.log("Waiting until OnReady event is fired")
else:
if self.ghostInit.receivers(SIGNAL("dom_is_ready(bool)")) > 0:
self.ghostInit.dom_is_ready.disconnect(self._page_loaded)
#Logger.log("Waiting until OnLoad event is fired")
body = QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl(QNetworkRequest.AlwaysNetwork)
for header in headers:
request.setRawHeader(header, headers[header])
if auth is not None:
self.network_manager.setAuthCredentials(auth[0], auth[1])
self._reset_for_loading()
self.main_frame.load(request, method, body)
if not wait_for_loading:
return self.get_loaded_page()
return self.wait_for_page_loaded()
def downloadBinPackage(packageUrl, destinationFileName):
request = QNetworkRequest(QUrl(packageUrl))
request.setRawHeader('Accept-Encoding', 'gzip,deflate')
reply = QgsNetworkAccessManager.instance().get(request)
evloop = QEventLoop()
reply.finished.connect(evloop.quit)
evloop.exec_(QEventLoop.ExcludeUserInputEvents)
content_type = reply.rawHeader('Content-Type')
if bytearray(content_type) == bytearray('application/zip'):
if os.path.isfile(destinationFileName):
os.unlink(destinationFileName)
destinationFile = open(destinationFileName, 'wb')
destinationFile.write(bytearray(reply.readAll()))
destinationFile.close()
else:
ret_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
raise IOError("{} {}".format(ret_code, packageUrl))
def ask_for_token(c_id, c_secret, request_status=True):
"""Ask a token from Isogeo API authentification page.
This send a POST request to Isogeo API with the user id and secret in
its header. The API should return an access token
"""
headervalue = "Basic " + base64.b64encode(c_id + ":" + c_secret)
data = urllib.urlencode({"grant_type": "client_credentials"})
databyte = QByteArray()
databyte.append(data)
url = QUrl('https://id.api.isogeo.com/oauth/token')
request = QNetworkRequest(url)
request.setRawHeader("Authorization", headervalue)
if request_status is True:
request_status = False
return request, databyte
QgsMessageLog.logMessage("Authentication succeeded", "Isogeo")
def doRequest(self,
url,
baseurl=None,
wait_time=None,
viewport=None,
js_source=None,
js_profile=None,
console=False):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.js_source = js_source
self.js_profile = js_profile
self.console = console
self.viewport = defaults.VIEWPORT if viewport is None else viewport
request = QNetworkRequest()
request.setUrl(QUrl(url.decode('utf8')))
if self.viewport != 'full':
# viewport='full' can't be set if content is not loaded yet
self._setViewportSize(self.viewport)
if getattr(self.splash_request, 'pass_headers', False):
headers = self.splash_request.getAllHeaders()
for name, value in headers.items():
request.setRawHeader(name, value)
if name.lower() == 'user-agent':
self.web_page.custom_user_agent = value
if baseurl:
self._baseUrl = QUrl(baseurl.decode('utf8'))
request.setOriginatingObject(self.web_page.mainFrame())
self._reply = self.network_manager.get(request)
self._reply.finished.connect(self._requestFinished)
else:
self.web_page.loadFinished.connect(self._loadFinished)
if self.splash_request.method == 'POST':
self.web_page.mainFrame().load(
request, QNetworkAccessManager.PostOperation,
self.splash_request.content.getvalue())
else:
self.web_page.mainFrame().load(request)
def autoSuggest(self):
# search type, language and search url
searchServer = self.settings.value(
"services/searchserver",
"https://api3.geo.admin.ch/rest/services/api/SearchServer")
searchType = self.settings.value("searchtype", "locations")
searchLanguage = self.settings.value("options/language", "de")
if searchType == "layers":
suggestUrl = searchServer + "?lang=" + searchLanguage + "&type=" + searchType + "&searchText="
elif searchType == "locations":
suggestUrl = searchServer + "?type=" + searchType + "&searchText="
elif searchType == "featuresearch":
featureLayerNames = ','.join(self.geoadminLayers)
suggestUrl = searchServer + "?features=" + featureLayerNames + "&type=" + searchType + "&searchText="
# http headers
headerFields = self.settings.value("options/headerfields")
headerValues = self.settings.value("options/headervalues")
headers = []
if headerFields and headerValues:
for i in range(len(headerFields)):
headers.append([headerFields[i], headerValues[i]])
# complete search url
# try to get the prefix/origins for location search
searchString = self.text()
listSearchString = searchString.split(' ')
originPrefix = ""
if listSearchString[0] in self.origins.keys():
originPrefix += listSearchString[0]
searchString = ' '.join(listSearchString[1:])
url = str(suggestUrl) + searchString
if originPrefix <> "":
url += "&origins=" + originPrefix.strip()
request = QNetworkRequest(QUrl(url))
for header in headers:
request.setRawHeader(header[0], header[1])
self.networkManager.get(request)
def renderpdf(url, dest):
app = QApplication(sys.argv)
web = QWebView()
request = QNetworkRequest(QUrl(url))
request.setRawHeader("X-Rendering-PDF", "true")
web.load(request)
printer = QPrinter()
printer.setPageSize(QPrinter.A4)
printer.setOutputFormat(QPrinter.PdfFormat)
printer.setOutputFileName(dest)
def print_file():
web.print_(printer)
QApplication.exit()
QObject.connect(web, SIGNAL("loadFinished(bool)"), print_file)
app.exec_()
def cbUserData(self, data):
self.currentUserData = data
if self.toolbar.avatarImage is None:
manager = QNetworkAccessManager()
manager.finished.connect(self.returnAvatar)
if 's3.amazonaws.com' in data['avatar_url']:
imageUrl = QUrl(data['avatar_url'])
else:
imageUrl = QUrl('http:' + data['avatar_url'])
request = QNetworkRequest(imageUrl)
request.setRawHeader('User-Agent', 'QGIS 2.x')
reply = manager.get(request)
loop = QEventLoop()
reply.finished.connect(loop.exit)
loop.exec_()
self.setUpUserData()
def ask_for_token(self, c_id, c_secret):
"""Ask a token from Isogeo API authentification page.
This send a POST request to Isogeo API with the user id and secret in
its header. The API should return an access token
"""
headervalue = "Basic " + base64.b64encode(c_id + ":" + c_secret)
data = urlencode({"grant_type": "client_credentials"})
databyte = QByteArray()
databyte.append(data)
url = QUrl('https://id.api.isogeo.com/oauth/token')
request = QNetworkRequest(url)
request.setRawHeader("Authorization", headervalue)
if self.requestStatusClear is True:
self.requestStatusClear = False
token_reply = self.manager.post(request, databyte)
token_reply.finished.connect(
partial(self.handle_token, answer=token_reply))
QgsMessageLog.logMessage("Authentication succeeded", "Isogeo")
def loadTile(self, x, y, zoom, url):
if self._manager is None:
self._manager = QNetworkAccessManager(parent=self)
self._manager.finished.connect(self.handleNetworkData)
self._cache = MapTileHTTPCache(maxSize=self._cacheSize, parent=self)
key = (x, y, zoom)
url = QUrl(url)
if url in self._cache:
# print('from cache')
data = self._cache[url]
self.tileLoaded.emit(x, y, zoom, data)
elif key in self._tileInDownload:
# Image is already in download... return
return
else:
request = QNetworkRequest(url=url)
request.setRawHeader('User-Agent', self._userAgent)
request.setAttribute(QNetworkRequest.User, key)
self._tileInDownload[key] = self._manager.get(request)
def open(self,
address,
method='get',
headers=None,
auth=None,
body=None,
default_popup_response=None):
"""Opens a web page.
@param address: The resource URL.
@param method: The Http method.
@param headers: An optional dict of extra request hearders.
@param auth: An optional tupple of HTTP auth (username, password).
@param body: An optional string containing a payload.
@param default_popup_response: the default response for any confirm/
alert/prompt popup from the Javascript (replaces the need for the with
blocks)
"""
headers = headers or {}
body = body or QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl = 0
for header in headers:
request.setRawHeader(header, headers[header])
self._auth = auth
self._auth_attempt = 0 # Avoids reccursion
self.page.mainFrame().load(request, method, body)
self._loaded = False
if default_popup_response is not None:
self._prompt_expected = (default_popup_response, None)
self._confirm_expected = (default_popup_response, None)
return self.wait_for_page_loaded()
def open(self, url, post_data=None, headers={}, receiver=None):
request = QNetworkRequest(QUrl(url))
if post_data:
debug.debug('post %s' % url)
postData = QByteArray()
for key, var in post_data.items():
postData.append('%s=%s&' % (key, var))
for header, var in headers.items():
request.setRawHeader(header, var)
reply = self.netWorkMgr.post(request, postData)
else:
debug.debug('get %s' % url)
for header, var in headers.items():
request.setRawHeader(header, var)
reply = self.netWorkMgr.get(request)
if receiver:
reply.finished.connect(receiver)
else:
receiver = self.netWorkMgr.parent()
reply.finished.connect(receiver.replyFinished)
def _sendRequest(self, url, params, headers={}):
if self.asynchonous:
if self.reply is not None:
self.reply.finished.disconnect(self.replyFinished)
self.reply.abort()
self.reply = None
url = QUrl(url)
for key, value in params.iteritems():
url.addQueryItem(key, value)
QgsLogger.debug('Request: {}'.format(url.toEncoded()))
request = QNetworkRequest(url)
for key, value in headers.iteritems():
request.setRawHeader(key, value)
self.reply = QgsNetworkAccessManager.instance().get(request)
self.reply.finished.connect(self.replyFinished)
else:
response = urllib2.urlopen(self.url + '?' +
urllib.urlencode(params))
data = json.load(response)
self.loadData(data)
def open(self, address, method='get', headers={}, auth=None):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tupple of HTTP auth (username, password).
:return: Page resource, All loaded resources.
"""
body = QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl(0)
for header in headers:
request.setRawHeader(header, headers[header])
self._auth = auth
self._auth_attempt = 0 # Avoids reccursion
self.main_frame.load(request, method, body)
self.loaded = False
return self.wait_for_page_loaded()
def request( url, params=None, secure=False, extraHeaders=None, successHandler=None, failureHandler=None, finishedHandler=None, parent=None ):
global nam, _isSecureSSL
if _isSecureSSL==False and False: #Warn the user of a potential security risk
msgRes = QtGui.QMessageBox.warning( None, QtCore.QCoreApplication.translate("NetworkService", "Insecure connection"),
QtCore.QCoreApplication.translate("Updater", "The cacerts.pem file is missing or invalid. Your passwords and data will be send unsecured! Continue without encryption? If unsure, choose \"abort\"!"),
QtCore.QCoreApplication.translate("NetworkService", "Continue in unsecure mode"),
QtCore.QCoreApplication.translate("NetworkService", "Abort") )
if msgRes==0:
_isSecureSSL=None
else:
sys.exit(1)
if secure:
key=securityTokenProvider.getKey()
if not params:
params = {}
params["skey"] = key
if url.lower().startswith("http"):
reqURL = QUrl(url)
else:
reqURL = QUrl( NetworkService.url+url )
req = QNetworkRequest( reqURL )
if extraHeaders:
for k, v in extraHeaders.items():
req.setRawHeader( k, v )
if params:
if isinstance( params, dict):
multipart, boundary = NetworkService.genReqStr( params )
req.setRawHeader( "Content-Type", b'multipart/form-data; boundary='+boundary+b'; charset=utf-8')
elif isinstance( params, bytes ):
req.setRawHeader( "Content-Type", b'application/x-www-form-urlencoded' )
multipart = params
else:
print( params )
print( type( params ) )
return( RequestWrapper( nam.post( req, multipart ), url=url ).getResult() )
else:
return( RequestWrapper( nam.get( req ), url=url).getResult() )
def request(self,
url,
user_agent='Mozilla',
cookies={},
timeout=15,
method='get',
data=None,
headers={}):
url_info = urlsplit(url)
self.resource_list = []
loop = QEventLoop()
self.view.loadFinished.connect(loop.quit)
# Timeout
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(loop.quit)
timer.start(timeout * 1000)
# User-Agent
self.page.user_agent = user_agent
# Cookies
cookie_obj_list = []
for name, value in cookies.items():
domain = ('.' + url_info.netloc).split(':')[0]
#print 'CREATE COOKIE %s=%s' % (name, value)
#print 'DOMAIN = %s' % domain
cookie_obj = QNetworkCookie(name, value)
cookie_obj.setDomain(domain)
cookie_obj_list.append(cookie_obj)
self.cookie_jar.setAllCookies(cookie_obj_list)
# Method
method_obj = getattr(QNetworkAccessManager,
'%sOperation' % method.capitalize())
# Ensure that Content-Type is correct if method is post
if method == 'post':
headers['Content-Type'] = 'application/x-www-form-urlencoded'
# Post data
if data is None:
data = QByteArray()
# Request object
request_obj = QNetworkRequest(QUrl(url))
# Headers
for name, value in headers.items():
request_obj.setRawHeader(name, value)
# Make a request
self.view.load(request_obj, method_obj, data)
loop.exec_()
if timer.isActive():
request_resource = None
url = str(self.page.mainFrame().url().toString()).rstrip('/')
for res in self.resource_list:
if url == res.url or url == res.url.rstrip('/'):
request_resource = res
break
if request_resource:
return self.build_response(request_resource)
else:
raise KitError('Request was successfull but it is not possible '\
'to associate the request to one of received responses')
else:
raise KitError('Timeout while loading %s' % url)
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None,
blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k, v)
# if self.authid:
# self.msg_log("Update request w/ authid: {0}".format(self.authid))
# QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, file):
body = body.read()
self.reply = func(req, body)
else:
self.reply = func(req)
# if self.authid:
# self.msg_log("Update reply w/ authid: {0}".format(self.authid))
# QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)
# necessary to trap local timout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(
self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
def get(self):
request = QNetworkRequest(self.url)
request.setRawHeader('User-Agent', osDetector.getOs() + " - UDS Connector " + VERSION)
self._manager.get(request)
def get(self, url):
future = Future()
url = QUrl(url)
request = QNetworkRequest(url)
request.setRawHeader(b"User-Agent", b"OWImageViewer/1.0")
request.setAttribute(
QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache
)
# Future yielding a QNetworkReply when finished.
reply = self._netmanager.get(request)
future._reply = reply
@future.add_done_callback
def abort_on_cancel(f):
# abort the network request on future.cancel()
if f.cancelled() and f._reply is not None:
f._reply.abort()
n_redir = 0
def on_reply_ready(reply, future):
nonlocal n_redir
# schedule deferred delete to ensure the reply is closed
# otherwise we will leak file/socket descriptors
reply.deleteLater()
future._reply = None
if reply.error() == QNetworkReply.OperationCanceledError:
# The network request was cancelled
reply.close()
future.cancel()
return
if reply.error() != QNetworkReply.NoError:
# XXX Maybe convert the error into standard
# http and urllib exceptions.
future.set_exception(Exception(reply.errorString()))
reply.close()
return
# Handle a possible redirection
location = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if location is not None and n_redir < 1:
n_redir += 1
location = reply.url().resolved(location)
# Retry the original request with a new url.
request = QNetworkRequest(reply.request())
request.setUrl(location)
newreply = self._netmanager.get(request)
future._reply = newreply
newreply.finished.connect(
partial(on_reply_ready, newreply, future))
reply.close()
return
reader = QImageReader(reply)
image = reader.read()
reply.close()
if image.isNull():
future.set_exception(Exception(reader.errorString()))
else:
future.set_result(image)
reply.finished.connect(partial(on_reply_ready, reply, future))
return future
def load_url_async(url):
m = QgsNetworkAccessManager.instance()
req = QNetworkRequest(QUrl(url))
req.setRawHeader('User-Agent', 'Magic Browser')
reply = m.get(req)
return reply
class NetworkMixin(object):
"""A mixin that can be used to send network request and receive replies."""
request_url = str
error = None
results = None
def __init__(self, request_url, geojson=False):
self.request_url = request_url
self.manager = QgsNetworkAccessManager.instance()
self.reply = None
self.url = QUrl(self.request_url)
self.req = QNetworkRequest(self.url)
self.results = QByteArray()
self.auth_token = get_authtoken()
# Paginated data
self._pagination_exhausted = False
self.geojson = geojson
if geojson:
self.combine_new_data = self.combine_new_geojson_data
def cancel_request(self):
"""Abort the request."""
self.reply.abort()
def connect_request(self):
"""Process the request."""
LOGGER.info('Requesting "%s"', self.request_url)
self.reply.readyRead.connect(self.connection_read_data)
self.reply.finished.connect(self.connection_finished)
self.reply.error.connect(self.connection_error)
def connect_paginated_request(self, prev_data, callback):
"""Process the request of a paginated resource."""
LOGGER.info('Requesting "%s" (paginated)', self.request_url)
self.reply.readyRead.connect(self.connection_read_data)
self.reply.finished.connect(
functools.partial(self._handle_paginated_data, prev_data,
callback))
self.reply.error.connect(self.connection_error)
def connect_get(self):
"""Send get request."""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.reply = self.manager.get(self.req)
self.connect_request()
def connect_post(self, data):
"""Send post request.
:param data: Context data to use with template
:type data: QByteArray
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.reply = self.manager.post(self.req, data)
self.connect_request()
def connect_json_post(self, data):
"""Send post request with json string.
:param data: Json string data
:type data: str
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.req.setRawHeader("Content-Type", "application/json")
self.reply = self.manager.post(self.req, data)
self.connect_request()
def connect_json_put(self, data):
"""Send put request with json string.
:param data: Json string data
:type data: str
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.req.setRawHeader("Content-Type", "application/json")
json_string = QByteArray(data)
p_buffer = QBuffer(self.manager)
p_buffer.setData(json_string)
self.reply = self.manager.sendCustomRequest(self.req, 'PUT', p_buffer)
self.connect_request()
def connect_json_patch(self, data):
"""Send patch request with json string.
:param data: Json string data
:type data: str
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.req.setRawHeader("Content-Type", "application/json")
json_string = QByteArray(data)
p_buffer = QBuffer(self.manager)
p_buffer.setData(json_string)
self.reply = self.manager.sendCustomRequest(self.req, 'PATCH',
p_buffer)
self.connect_request()
def connect_get_paginated(self, prev_data=None, callback=None):
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.reply = self.manager.get(self.req)
self.connect_paginated_request(
prev_data or self.results, callback
or self._set_results_and_complete)
def _handle_paginated_data(self, prev_data, callback):
"""
Once all data from single request is returned, merge it with
previous responses and follow pagination.
"""
# Add new request's data to prev_data
resp_txt = str(self.results)
merged_data = self.combine_new_data(str(prev_data or ''), resp_txt)
merged_data = QByteArray(merged_data)
# Get next page
next_url = None
try:
next_url = json.loads(resp_txt).get('next')
except ValueError:
pass
if next_url:
next_req = NetworkMixin(next_url, geojson=self.geojson)
return next_req.connect_get_paginated(prev_data=merged_data,
callback=callback)
LOGGER.debug('Pagination finished at "%s"', self.request_url)
callback(merged_data)
def _set_results_and_complete(self, results):
self.results = results
self._pagination_exhausted = True
self.connection_finished()
@staticmethod
def combine_new_data(old_data_str, new_data_str):
"""
Merge results from new page of data with results from previous
pages of data.
Expects a response in a format such as:
{
"count": 10,
"next": null,
"previous": null,
"results": [ ... ]
}
"""
old_data = ''
new_data = ''
try:
old_data = json.loads(old_data_str or '[]')
new_data = json.loads(new_data_str)
except ValueError:
pass
results_new_data = []
if 'results' in new_data:
results_new_data = new_data['results']
return json.dumps(old_data + results_new_data)
@staticmethod
def combine_new_geojson_data(old_data_str, new_data_str):
"""
Merge results from new page of GeoJSON data with results from
previous pages of data.
Expects a response in a format such as:
{
"count": 10,
"next": null,
"previous": null,
"results": {
"type": "FeatureCollection",
"features": [ ... ]
}
}
"""
old_data = dict()
new_data = dict()
try:
old_data = json.loads(old_data_str or '[]')
new_data = json.loads(new_data_str)
except ValueError:
pass
if not old_data and 'results' in new_data:
return json.dumps(new_data['results'])
if 'features' in old_data and 'results' in new_data:
old_data['features'] += new_data['results']['features']
return json.dumps(old_data)
def connection_read_data(self):
"""Get data from self.reply and append it to results."""
self.results += self.reply.readAll()
def get_json_results(self):
"""Convert results to json object."""
return json.loads(str(self.results))
def connection_error(self):
"""Handle error connection."""
error_result = self.reply.error
try:
http_code = int(
self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute))
except TypeError:
http_code = None
self.http_code = 404
if error_result == QNetworkReply.UnknownNetworkError:
msg = 'The network is unreachable.'
elif error_result == QNetworkReply.ProtocolUnknownError \
or error_result == QNetworkReply.HostNotFoundError:
msg = 'Host not found : %s' % self.url.encodedHost()
else:
if http_code:
msg = 'Error code:' + str(http_code)
self.http_code = http_code
else:
msg = 'Can\'t find the server'
LOGGER.debug(msg)
self.error = msg
@abc.abstractmethod
def connection_finished(self):
return
def is_finished(self, paginated=False):
return (self._pagination_exhausted
if paginated else self.reply.isFinished())
def _getRequest(self, url):
request = QNetworkRequest(url)
request.setRawHeader("Content-Type", "application/json")
request.setRawHeader('User-Agent', 'QGISCartoDB 0.2.x')
return request
def executeRequest(self, url, callback, post=None):
self.messageBar.clearWidgets()
self.cmbOfferings.setEnabled(False)
self.tabWidget.setEnabled(False)
if post:
request = QNetworkRequest(url)
request.setRawHeader('Content-Type', 'application/xml')
self.reply = QgsNetworkAccessManager.instance().post(request, post)
else:
self.reply = QgsNetworkAccessManager.instance().get(
QNetworkRequest(url))
progressMessageBar = self.messageBar.createMessage(
self.tr("Please wait while downloading"))
progressBar = QtGui.QProgressBar(self)
progressBar.setMinimum(0)
progressBar.setFormat(self.tr("%v bytes downloaded!"))
progressBar.setAlignment(QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
def updateProgress(read, total):
try:
progressBar.setMaximum(total)
progressBar.setValue(read)
except:
pass
updateProgress(0, 0)
self.reply.downloadProgress.connect(updateProgress)
progressMessageBar.layout().addWidget(progressBar)
btnAbort = QtGui.QPushButton(self.tr("Abort"))
btnAbort.clicked.connect(self.reply.abort)
progressMessageBar.layout().addWidget(btnAbort)
fd, replyFilename = mkstemp(suffix=".xml",
prefix=callback.__name__,
dir=self.selectedWorkDir())
os.close(fd)
replyFile = open(replyFilename, "w")
def replyReadyRead():
try:
replyFile.write(self.reply.readAll())
except:
pass
self.reply.readyRead.connect(replyReadyRead)
def finishRequest():
replyFile.close()
self.cmbOfferings.setEnabled(True)
self.tabWidget.setEnabled(True)
self.messageBar.clearWidgets()
if self.reply.error() != QNetworkReply.NoError:
self.messageBar.pushMessage(self.reply.errorString(),
QgsMessageBar.CRITICAL)
else:
callback(replyFilename)
self.reply.deleteLater()
self.reply.finished.connect(finishRequest)
self.messageBar.pushWidget(progressMessageBar, QgsMessageBar.INFO)
