def __downloadFile(self, url, filename, doneMethod = None):
"""
Private slot to download the given file.
@param url URL for the download (string or QString)
@param filename local name of the file (string or QString)
@param doneMethod method to be called when done
"""
self.__updateButton.setEnabled(False)
self.__downloadButton.setEnabled(False)
self.__downloadCancelButton.setEnabled(True)
self.statusLabel.setText(url)
self.__doneMethod = doneMethod
self.__downloadURL = url
self.__downloadFileName = QString(filename)
self.__downloadIODevice = QFile(self.__downloadFileName + ".tmp")
self.__downloadCancelled = False
request = QNetworkRequest(QUrl(url))
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.AlwaysNetwork)
reply = self.__networkManager.get(request)
self.connect(reply, SIGNAL("finished()"), self.__downloadFileDone)
self.connect(reply, SIGNAL("downloadProgress(qint64, qint64)"),
self.__downloadProgress)
self.__replies.append(reply)
def _wrap_request(self, request):
req = QNetworkRequest(request)
req_id = next(self._request_ids)
req.setAttribute(self._REQUEST_ID, req_id)
if hasattr(request, 'timeout'):
req.timeout = request.timeout
return req, req_id
def open(self, address, method='get', headers={}, auth=None, body=None):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tupple of HTTP auth (username, password).
:param body: An optional string containing a payload.
:return: Page resource, All loaded resources.
"""
body = body or QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl(0)
for header in headers:
request.setRawHeader(header, headers[header])
self._auth = auth
self._auth_attempt = 0 # Avoids reccursion
self.main_frame.load(request, method, body)
self.loaded = False
return self.wait_for_page_loaded()
def make_request(self, url):
url = QUrl(url)
req = QNetworkRequest(url)
for header in self.headers:
val = self.headers[header]
req.setRawHeader(header, val)
return req
def _doUpload(self):
if self.current_track is None or self._aborted:
return
track, track_type = self.current_track
self.statusMessage.emit('Uploading {0}'.format(track['name']))
log.debug('Sending upload request (%s)', track['name'])
# data_fields, data = bryton_gpx_to_strava_json(track['gpx'])
data = tcx.bryton_gpx_to_tcx(track['gpx'], activity_type=track_type, pretty=False, device=self.device_info)
req = QNetworkRequest(QUrl(self.UPLOAD))
req.setHeader(QNetworkRequest.ContentTypeHeader,
"application/json")
# d2 = json.loads(open('tmp/test.json').read())
# d2['token'] = self.auth_token
# self.reply = self.network_manager.post(req, json.dumps(d2))
self.reply = self.network_manager.post(req, json.dumps({
'token' : self.auth_token,
'type' : 'tcx',
'data' : data,
'activity_type' : track_type,
}))
self.reply.finished.connect(self._onUploaded)
def read(self, **kwargs):
self.connect(self.manager, SIGNAL("finished( QNetworkReply* )"), self.readRequestFinished)
params = []
try:
queryable = kwargs["queryable"]
params.append({"queryable": queryable})
queryableList = QString(queryable).split(",")
except KeyError:
pass
for key, value in kwargs.items():
if QString(key).split("__")[0] in queryableList:
params.append({key: value})
url = "%s/stakeholders?" % self.host
for p in params:
url = "%s%s=%s&" % (url, p.keys()[0], p.values()[0])
qUrl = QUrl(url)
# Create a new request
request = QNetworkRequest(qUrl)
request.setRawHeader("Authorization", self.login)
request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json")
self.manager.get(request)
return url
def open(self, address, method='get', headers={}, auth=None):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tupple of HTTP auth (Basic, username, password).
:return: Page resource, All loaded resources.
"""
body = QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
if auth is not None:
auth_type, username, password = auth
def authenticate(reply, authenticator):
authenticator.setUser(username)
authenticator.setPassword(password)
self.page.networkAccessManager().authenticationRequired.connect(authenticate)
if not "User-Agent" in headers:
headers["User-Agent"] = self.user_agent
for header in headers:
request.setRawHeader(header, headers[header])
self.main_frame.load(request, method, body)
self.loaded = False
return self.wait_for_page_loaded()
def open(self, address, method='get', headers={}, auth=None, body=None,
default_popup_response=None):
"""Opens a web page.
:param address: The resource URL.
:param method: The Http method.
:param headers: An optional dict of extra request hearders.
:param auth: An optional tuple of HTTP auth (username, password).
:param body: An optional string containing a payload.
:param default_popup_response: the default response for any confirm/
alert/prompt popup from the Javascript (replaces the need for the with
blocks)
:return: Page resource, All loaded resources.
"""
body = body or QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Error("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl(0)
for header in headers:
request.setRawHeader(header, headers[header])
self._auth = auth
self._auth_attempt = 0 # Avoids reccursion
self.main_frame.load(request, method, body)
self.loaded = False
Ghost._prompt_expected = (default_popup_response, None)
Ghost._confirm_expected = (default_popup_response, None)
return self.wait_for_page_loaded()
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('q', query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def createRequest(self, operation, request, outgoingData=None):
request = QNetworkRequest(request)
old_proxy = self.proxy()
# Hack to change Language
request.setRawHeader("Accept-Language","fr-FR,fr;q=0.8");
#request.setRawHeader("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
#request.setRawHeader("Connection","keep-alive");
splash_proxy_factory = self._getSplashProxyFactory(request)
if splash_proxy_factory:
proxy_query = QNetworkProxyQuery(request.url())
proxy = splash_proxy_factory.queryProxy(proxy_query)[0]
self.setProxy(proxy)
# this method is called createRequest, but in fact it creates a reply
reply = super(ProxiedQNetworkAccessManager, self).createRequest(
operation, request, outgoingData
)
reply.error.connect(self._handleError)
reply.finished.connect(self._handleFinished)
reply.metaDataChanged.connect(self._handleMetaData)
reply.downloadProgress.connect(self._handleDownloadProgress)
self.setProxy(old_proxy)
return reply
def request(self, iri, method="GET", extra_headers=None,
oauth_credentials=None, post_content=None):
"""Return a deferred that will be fired with a Response object."""
uri = self.iri_to_uri(iri)
request = QNetworkRequest(QUrl(uri))
request.setSslConfiguration(self.ssl_config)
headers = yield self.build_request_headers(uri, method, extra_headers,
oauth_credentials)
for key, value in headers.items():
request.setRawHeader(key, value)
post_buffer = QBuffer()
post_buffer.setData(post_content)
try:
result = yield self._perform_request(request, method, post_buffer)
except ProxyUnauthorizedError as e:
app_proxy = QNetworkProxy.applicationProxy()
proxy_host = app_proxy.hostName() if app_proxy else "proxy server"
got_creds = yield self.request_proxy_auth_credentials(
proxy_host, self.proxy_retry)
if got_creds:
self.proxy_retry = True
result = yield self.request(iri, method, extra_headers,
oauth_credentials, post_content)
else:
excp = WebClientError('Proxy creds needed.', e)
defer.returnValue(excp)
defer.returnValue(result)
def __init__(self, url, baseurl=None):
self.url = url
self.web_view = QWebView()
self.network_manager = SplashQNetworkAccessManager()
self.web_page = SplashQWebPage()
self.web_page.setNetworkAccessManager(self.network_manager)
self.web_view.setPage(self.web_page)
self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)
settings = self.web_view.settings()
settings.setAttribute(QWebSettings.JavascriptEnabled, True)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.deferred = defer.Deferred()
request = QNetworkRequest()
request.setUrl(QUrl(url))
if baseurl:
self._baseUrl = QUrl(baseurl)
self.network_manager.finished.connect(self._requestFinished)
self.network_manager.get(request)
else:
self.web_page.loadFinished.connect(self._loadFinished)
self.web_page.mainFrame().load(request)
def checkWeather(self):
if self.reply is not None:
return
request = QNetworkRequest()
request.setUrl(QUrl("http://api.liqwei.com/weather/"))
self.reply = self.http.get(request)
self.reply.finished.connect(self.onReplyFinished)
self.reply.error.connect(self.onReplyError)
def prepare_request(self, url_string):
url = QUrl(url_string)
request = QNetworkRequest()
request.setUrl(url)
request.setRawHeader('User-Agent', 'Cuivre Web Browser')
return request
def post(self, mgr , url , datas):
request = QNetworkRequest(QtCore.QUrl(url))
request.setHeader(QNetworkRequest.ContentTypeHeader,"application/x-www-form-urlencoded")
self.reply = mgr.post(request , datas)
self.reply.finished.connect(self.replyfinished)
self.reply.readyRead.connect(self.readyRead)
return self.reply
def get_data(self):
request = QNetworkRequest(QUrl(self.config.url))
request.setHeader(QNetworkRequest.ContentTypeHeader, 'text/xml') # or? "text/xml; charset=utf-8"
reply = self.network_manager.post(request, self.xml())
extension = '.zip'
if self.config.jrodos_format == 'application/json':
extension = '.json'
filename = self.config.output_dir + '/' + unicode(self.column) + '_' + unicode(
self.config.jrodos_verticals) + extension
reply.finished.connect(partial(self._data_retrieved, reply, filename))
def get(self, url):
future = Future()
url = url = QUrl(url)
request = QNetworkRequest(url)
request.setRawHeader("User-Agent", "OWImageViewer/1.0")
request.setAttribute(
QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache
)
# Future yielding a QNetworkReply when finished.
reply = self._netmanager.get(request)
future._reply = reply
n_redir = 0
def on_reply_ready(reply, future):
nonlocal n_redir
if reply.error() == QNetworkReply.OperationCanceledError:
# The network request itself was canceled
future.cancel()
return
if reply.error() != QNetworkReply.NoError:
# XXX Maybe convert the error into standard
# http and urllib exceptions.
future.set_exception(Exception(reply.errorString()))
return
# Handle a possible redirection
location = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if location is not None and n_redir < 1:
n_redir += 1
print(location)
location = reply.url().resolved(location)
# Retry the original request with a new url.
request = QNetworkRequest(reply.request())
request.setUrl(location)
newreply = self._netmanager.get(request)
future._reply = newreply
newreply.finished.connect(
partial(on_reply_ready, newreply, future))
return
reader = QImageReader(reply)
image = reader.read()
if image.isNull():
future.set_exception(Exception(reader.errorString()))
else:
future.set_result(image)
reply.finished.connect(partial(on_reply_ready, reply, future))
return future
def sendVote(self, plugin_id, vote):
""" send vote via the RPC """
if not plugin_id or not vote:
return False
url = "http://plugins.qgis.org/plugins/RPC2/"
params = "{\"id\":\"djangorpc\",\"method\":\"plugin.vote\",\"params\":[%s,%s]}" % (str(plugin_id), str(vote))
req = QNetworkRequest(QUrl(url))
req.setRawHeader("Content-Type", "application/json")
QgsNetworkAccessManager.instance().post(req, params)
return True
def request_obj(self, url, headers=None):
""" Return a QNetworkRequest object """
request = QNetworkRequest()
request.setUrl(QUrl(url))
request.setOriginatingObject(self.web_page.mainFrame())
if headers is not None:
self.web_page.skip_custom_headers = True
self._set_request_headers(request, headers)
return request
def __init__(self, parent=None):
# super(MainWindow, self).__init__(parent)
super(MainWindow, self).__init__(parent)
self.ui = Ui_Dialog()
self.ui.setupUi(self)
request = QNetworkRequest()
request.setUrl(QUrl('http://www.apple.com/cn/retail/shanghaiiapm/'))
reply = self.manager.get(request)
reply.finished.connect(self.replyFinished)
self.ui.webView.load(request)
def executeRequest (self, url, callback, post = None):
self.messageBar.clearWidgets()
self.cmbOfferings.setEnabled (False)
self.tabWidget.setEnabled (False)
if post:
request = QNetworkRequest(url)
request.setRawHeader('Content-Type', 'application/xml')
self.reply = QgsNetworkAccessManager.instance().post(request,post)
else:
self.reply = QgsNetworkAccessManager.instance().get(QNetworkRequest(url))
progressMessageBar = self.messageBar.createMessage(self.tr("Please wait while downloading"))
progressBar = QtGui.QProgressBar(self)
progressBar.setMinimum(0)
progressBar.setFormat(self.tr("%v bytes downloaded!"))
progressBar.setAlignment(QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
def updateProgress (read, total):
try:
progressBar.setMaximum (total)
progressBar.setValue (read)
except: pass
updateProgress (0,0)
self.reply.downloadProgress.connect (updateProgress)
progressMessageBar.layout().addWidget(progressBar)
btnAbort = QtGui.QPushButton(self.tr("Abort"))
btnAbort.clicked.connect(self.reply.abort)
progressMessageBar.layout().addWidget(btnAbort)
fd, replyFilename = mkstemp(suffix=".xml", prefix = callback.__name__, dir = self.selectedWorkDir())
os.close(fd)
replyFile = open (replyFilename, "w")
def replyReadyRead ():
try: replyFile.write (self.reply.readAll())
except: pass
self.reply.readyRead.connect(replyReadyRead)
def finishRequest ():
replyFile.close()
self.cmbOfferings.setEnabled (True)
self.tabWidget.setEnabled(True)
self.messageBar.clearWidgets()
if self.reply.error() != QNetworkReply.NoError:
self.messageBar.pushMessage(self.reply.errorString(), QgsMessageBar.CRITICAL)
else:
callback(replyFilename)
self.reply.deleteLater()
self.reply.finished.connect(finishRequest)
self.messageBar.pushWidget(progressMessageBar, QgsMessageBar.INFO)
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(
r'output="[a-z]*"', 'output="' + self.__output + '"', query)
query = re.sub(
r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
encoded_query = QUrl.toPercentEncoding(query)
url_query.addEncodedQueryItem('data', encoded_query)
url_query.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setPort(80)
proxy = get_proxy()
if proxy:
self.network.setProxy(proxy)
request = QNetworkRequest(url_query)
request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, self.data):
raise OverpassTimeoutException
else:
return self.data
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass API")
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
request = QNetworkRequest(QUrl(url))
request.setRawHeader("User-Agent", self.userAgent)
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self.replyFinished)
self.requestingUrls.append(url)
self.replies.append(reply)
return reply
def authenticate(self, username, password):
log.debug('Sending auth request')
req = QNetworkRequest(QUrl(self.LOGIN))
req.setHeader(QNetworkRequest.ContentTypeHeader,
"application/x-www-form-urlencoded")
self.reply = self.network_manager.post(req, urllib.urlencode({
'email' : username,
'password' : password,
}))
self.reply.finished.connect(self._onAuthenticated)
def doRequest(self, url, baseurl=None, wait_time=None):
self.url = url
self.wait_time = defaults.WAIT_TIME if wait_time is None else wait_time
self.deferred = defer.Deferred()
request = QNetworkRequest()
request.setUrl(QUrl(url))
if baseurl:
self._baseUrl = QUrl(baseurl)
self.network_manager.finished.connect(self._requestFinished)
self.network_manager.get(request)
else:
self.web_page.loadFinished.connect(self._loadFinished)
self.web_page.mainFrame().load(request)
class SettingsProtocol(QObject):
readSignal = pyqtSignal(bool, int, QString)
def __init__(self, host, user, password):
QObject.__init__(self)
# Set the host
self.host = host
# Create a base64 encoded credential string from user name and password.
# This is required for the HTTP basic access authentication, compare
# also http://en.wikipedia.org/wiki/Basic_access_authentication
self.userlogin = "******" % (user, password)
self.login = "******" + QByteArray(self.userlogin).toBase64()
# Create a new QNetworkAccessManager and connect it to the
# authenticationRequired signal
self.manager = QNetworkAccessManager(self)
# self.connect(self.manager, SIGNAL("authenticationRequired( QNetworkReply*, QAuthenticator* )"), self.slotAuthenticationRequired)
def read(self):
self.connect(self.manager, SIGNAL("finished( QNetworkReply* )"), self.readRequestFinished)
url = "%s/config/geometrytaggroups" % self.host
qUrl = QUrl(url)
self.request = QNetworkRequest(qUrl)
self.request.setRawHeader("Authorization", self.login)
self.request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json")
self.manager.get(self.request)
return url
def readRequestFinished(self, reply):
# Get the HTTP status code from the reply
self.httpStatusCode = int(reply.attribute(QNetworkRequest.HttpStatusCodeAttribute).toString())
data = reply.readAll()
# Check the status code see also http://en.wikipedia.org/wiki/HTTP_status_code
# In case of a successful upload we get a 201 status code back and the
# "stylePosted" signal is emitted with the first parameter set to True.
# If the query didn't succeed, the status code is 4xx indicating that
# the host was not found, the authentication failed or a forbidden
# action in case a style with the same name already exists. It's up to
# the receiver to handle these status codes.
self.readSignal.emit(self.httpStatusCode in (200, 201), self.httpStatusCode, QString(data))
def createRequest(self, op, request, outgoingData=None):
"""
Protected method to create a request.
@param op the operation to be performed (QNetworkAccessManager.Operation)
@param request reference to the request object (QNetworkRequest)
@param outgoingData reference to an IODevice containing data to be sent
(QIODevice)
@return reference to the created reply object (QNetworkReply)
"""
scheme = request.url().scheme()
if scheme == "https" and (not SSL_AVAILABLE or not QSslSocket.supportsSsl()):
return NetworkProtocolUnknownErrorReply(scheme, self)
if op == QNetworkAccessManager.PostOperation and outgoingData is not None:
outgoingDataByteArray = outgoingData.peek(1024 * 1024)
Helpviewer.HelpWindow.HelpWindow.passwordManager().post(request, outgoingDataByteArray)
reply = None
if unicode(scheme) in self.__schemeHandlers:
reply = self.__schemeHandlers[unicode(scheme)].createRequest(op, request, outgoingData)
if reply is not None:
return reply
if not self.__acceptLanguage.isEmpty():
req = QNetworkRequest(request)
req.setRawHeader("Accept-Language", self.__acceptLanguage)
else:
req = request
# AdBlock code
if op == QNetworkAccessManager.GetOperation:
if self.__adblockNetwork is None:
self.__adblockNetwork = Helpviewer.HelpWindow.HelpWindow.adblockManager().network()
reply = self.__adblockNetwork.block(req)
if reply is not None:
reply.setParent(self)
return reply
# set cache policy
if op == QNetworkAccessManager.GetOperation:
urlHost = req.url().host()
for host in self.NoCacheHosts:
if host in urlHost:
req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork)
break
else:
req.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork)
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
self.emit(
SIGNAL("requestCreated(QNetworkAccessManager::Operation, const QNetworkRequest&, QNetworkReply*)"),
op,
req,
reply,
)
return reply
def read(self, extent, iface):
self.connect(self.manager, SIGNAL("finished( QNetworkReply* )"), self.readRequestFinished)
#self.finished.connect(self.readRequestFinished)
# Get the map extent
xmin = extent.xMinimum()
ymin = extent.yMinimum()
xmax = extent.xMaximum()
ymax = extent.yMaximum()
# Get the authority identifier for the projects destination reference
# system. Normally in case of EPSG that authid looks like "EPSG:4326"
authid = iface.mapCanvas().mapRenderer().destinationCrs().authid()
authcode = int(authid.split(":")[-1])
# Add explicitly the EPSG code to the bounding box, to enable users to
# work in a local reference system
url = "%s/activities/json?bbox=%f,%f,%f,%f&epsg=%1.f" % (self.host, xmin, ymin, xmax, ymax, authcode)
qUrl = QUrl(url)
self.request = QNetworkRequest(qUrl)
self.request.setRawHeader("Authorization", self.login)
self.request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json")
self.manager.get(self.request)
return url
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
# create request
request = QNetworkRequest(QUrl(url))
if self.userAgent:
request.setRawHeader("User-Agent", self.userAgent) # will be overwritten in QgsNetworkAccessManager::createRequest() since 2.2
# send request
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self._replyFinished)
self.requestingReplies[url] = reply
return reply
def upload(self, filePath, returnDict=True):
self.returnDict = returnDict
file = QFile(filePath)
file.open(QFile.ReadOnly)
url = QUrl(self.apiUrl + "imports/?api_key={}".format(self.apiKey))
files = {"file": file}
multipart = self._createMultipart(files=files)
request = QNetworkRequest(url)
request.setHeader(QNetworkRequest.ContentTypeHeader, "multipart/form-data; boundary=%s" % multipart.boundary())
request.setRawHeader("User-Agent", "QGISCartoDB 0.2.x")
reply = self.manager.post(request, multipart)
loop = QEventLoop()
reply.uploadProgress.connect(self.progressCB)
reply.error.connect(self._error)
reply.finished.connect(loop.exit)
loop.exec_()
def open(self, url, post_data=None, headers={}, receiver=None):
request = QNetworkRequest(QUrl(url))
if post_data:
debug.debug('post %s' % url)
postData = QByteArray()
for key, var in post_data.items():
postData.append('%s=%s&' % (key, var))
for header, var in headers.items():
request.setRawHeader(header, var)
reply = self.netWorkMgr.post(request, postData)
else:
debug.debug('get %s' % url)
for header, var in headers.items():
request.setRawHeader(header, var)
reply = self.netWorkMgr.get(request)
if receiver:
reply.finished.connect(receiver)
else:
receiver = self.netWorkMgr.parent()
reply.finished.connect(receiver.replyFinished)
def download(self):
"""Downloading the file.
:returns: True if success, otherwise returns a tuple with format like
this (QNetworkReply.NetworkError, error_message)
:raises: IOError - when cannot create output_path
"""
# Prepare output path
self.output_file = QFile(self.output_path)
if not self.output_file.open(QFile.WriteOnly):
raise IOError(self.output_file.errorString())
# Prepare downloaded buffer
self.downloaded_file_buffer = QByteArray()
# Request the url
request = QNetworkRequest(self.url)
self.reply = self.manager.get(request)
self.reply.readyRead.connect(self.get_buffer)
self.reply.finished.connect(self.write_data)
self.manager.requestTimedOut.connect(self.request_timeout)
if self.progress_dialog:
# progress bar
def progress_event(received, total):
"""Update progress.
:param received: Data received so far.
:type received: int
:param total: Total expected data.
:type total: int
"""
# noinspection PyArgumentList
QCoreApplication.processEvents()
self.progress_dialog.adjustSize()
human_received = humanize_file_size(received)
human_total = humanize_file_size(total)
label_text = tr(
"%s : %s of %s" %
(self.prefix_text, human_received, human_total))
self.progress_dialog.setLabelText(label_text)
self.progress_dialog.setMaximum(total)
self.progress_dialog.setValue(received)
# cancel
def cancel_action():
"""Cancel download."""
self.reply.abort()
self.reply.deleteLater()
self.reply.downloadProgress.connect(progress_event)
self.progress_dialog.canceled.connect(cancel_action)
# Wait until finished
# On Windows 32bit AND QGIS 2.2, self.reply.isFinished() always
# returns False even after finished slot is called. So, that's why we
# are adding self.finished_flag (see #864)
while not self.reply.isFinished() and not self.finished_flag:
# noinspection PyArgumentList
QCoreApplication.processEvents()
result = self.reply.error()
try:
http_code = int(
self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute))
except TypeError:
# If the user cancels the request, the HTTP response will be None.
http_code = None
self.reply.abort()
self.reply.deleteLater()
if result == QNetworkReply.NoError:
return True, None
elif result == QNetworkReply.UnknownNetworkError:
return False, tr(
'The network is unreachable. Please check your internet '
'connection.')
elif http_code == 408:
msg = tr('Sorry, the server aborted your request. '
'Please try a smaller area.')
LOGGER.debug(msg)
return False, msg
elif http_code == 509:
msg = tr(
'Sorry, the server is currently busy with another request. '
'Please try again in a few minutes.')
LOGGER.debug(msg)
return False, msg
elif result == QNetworkReply.ProtocolUnknownError or \
result == QNetworkReply.HostNotFoundError:
LOGGER.exception('Host not found : %s' % self.url.encodedHost())
return False, tr(
'Sorry, the server is unreachable. Please try again later.')
elif result == QNetworkReply.ContentNotFoundError:
LOGGER.exception('Path not found : %s' % self.url.path())
return False, tr('Sorry, the layer was not found on the server.')
else:
return result, self.reply.errorString()
def createRequest(self, operation, request, data):
if request.url().scheme() != "tentia":
return QNetworkAccessManager.createRequest(self, operation, request, data)
else:
self.tentia_callback(request.url())
return QNetworkAccessManager.createRequest(self, QNetworkAccessManager.GetOperation, QNetworkRequest(QtCore.QUrl()))
def send_debugger_command(self, command):
if self.debug_port is None:
print "No debugger port"
return
url = QUrl()
url.setScheme("http")
url.setHost(CON.address)
url.setPort(self.debug_port)
url.setPath("/debugger")
print("[VDBG] ' %s ' Command Sent" % command)
data = {}
request = QNetworkRequest(url)
if command == "bb":
data['clear'] = True
data['add'] = []
bpCnt = len(self.editorManager.bp_info[1])
for r in range(0, bpCnt):
bp_info = self.editorManager.bp_info[2][r]
n = re.search(":", bp_info).end()
fName = bp_info[:n - 1]
lNum = int(bp_info[n:]) - 1
if self.editorManager.bp_info[1][r] == "on":
bState = True
else:
bState = False
#data['add'].append({'file':fName, 'line':lNum, 'on':bState})
bp_item = {}
bp_item['file'] = fName
bp_item['line'] = lNum + 1
bp_item['on'] = bState
data['add'].append(bp_item)
params = json.dumps(data)
reply = self.manager.post(request, command + ' ' + params)
reply.command = command
else:
reply = self.manager.post(request, command)
reply.command = command
def debugger_reply_finished(reply):
def foo():
if reply.error() == QNetworkReply.NoError:
print("[VDBG] ' %s ' Response" % reply.command)
if reply.command == "bn":
return
if reply.command == "r":
self.main.deviceManager.socket.write('/close\n\n')
self.main.rSent = False
if self.main.onExit == True:
if self.editorManager.tab != None:
while self.editorManager.tab.count() != 0:
self.editorManager.close()
self.stop()
self.main.close()
data = self.getFileLineInfo_Resp(str(reply.readAll()),
command)
if data is not None:
if reply.command == DBG_CMD_INFO:
self.inbreak = True
self.inspector.ui.refresh.setEnabled(False)
self.inspector.ui.search.setEnabled(False)
# Open File, Show Current Lines
if self.file_name.startswith("/"):
self.file_name = self.file_name[1:]
self.file_name = self.file_name + '/'
if self.file_name.endswith("/"):
self.file_name = self.file_name[:len(
self.file_name) - 1]
current_file = os.path.join(
str(self.main.path), str(self.file_name))
if self.current_debug_file != current_file:
self.editorManager.newEditor(
current_file, None, self.line_no,
self.current_debug_file, True)
else:
self.editorManager.newEditor(
current_file, None, self.line_no, None,
True)
self.current_debug_file = current_file
# Local Variable Table
local_info = self.getLocalInfo_Resp(data)
if local_info is not None:
self.debugWindow.populateLocalTable(local_info)
# Global Variable Table
global_info = self.getGlobalInfo_Resp(data)
if global_info is not None:
self.debugWindow.populateGlobalTable(
global_info, self.editorManager)
# Stack Trace Table
stack_info = self.getStackInfo_Resp(data)
if stack_info is not None:
self.backtraceWindow.populateTraceTable(
stack_info, self.editorManager)
#reply = None
#reply.command = None
# TODO: Here we should enable the debug UI
self.main.debug_stepinto.setEnabled(True)
self.main.debug_stepover.setEnabled(True)
self.main.debug_stepout.setEnabled(False)
self.main.debug_pause_bt.setEnabled(False)
self.main.debug_continue_bt.setEnabled(True)
self.main.ui.actionContinue.setEnabled(True)
self.main.ui.actionPause.setEnabled(False)
self.main.ui.actionStep_into.setEnabled(True)
self.main.ui.actionStep_over.setEnabled(True)
self.main.ui.actionStep_out.setEnabled(False)
self.main.debug_run = False
elif reply.command[:
1] == DBG_CMD_BREAKPOINT or reply.command == DBG_CMD_BB or reply.command[:
1] == DBG_CMD_DELETE:
# Break Point
break_info = self.getBreakPointInfo_Resp(data)
if break_info is not None:
self.debugWindow.populateBreakTable(
break_info, self.editorManager)
if reply.command == DBG_CMD_BB:
return
editor = self.editorManager.app.focusWidget()
if editor is not None:
nline = editor.margin_nline
else:
index = self.editorManager.tab.currentIndex()
editor = self.editorManager.tab.editors[index]
nline = editor.margin_nline
if reply.command[:
1] == DBG_CMD_DELETE and nline is not None:
if editor.current_line != nline:
editor.markerDelete(nline, -1)
else:
editor.markerDelete(nline, -1)
editor.markerAdd(nline,
editor.ARROW_MARKER_NUM)
editor.line_click[nline] = 0
return
elif nline is None:
return
# Break Point Setting t
if not editor.line_click.has_key(
nline) or editor.line_click[nline] == 0:
if editor.current_line != nline:
editor.markerAdd(
nline, editor.ACTIVE_BREAK_MARKER_NUM)
else:
editor.markerDelete(
nline, editor.ARROW_MARKER_NUM)
editor.markerAdd(
nline,
editor.ARROW_ACTIVE_BREAK_MARKER_NUM)
editor.line_click[nline] = 1
# Break Point Deactivate
elif editor.line_click[nline] == 1:
if editor.current_line != nline:
editor.markerDelete(
nline, editor.ACTIVE_BREAK_MARKER_NUM)
editor.markerAdd(
nline,
editor.DEACTIVE_BREAK_MARKER_NUM)
else:
editor.markerDelete(
nline,
editor.ARROW_ACTIVE_BREAK_MARKER_NUM)
editor.markerAdd(
nline,
editor.ARROW_DEACTIVE_BREAK_MARKER_NUM)
editor.line_click[nline] = 2
# Break Point Active
elif editor.line_click[nline] == 2:
if editor.current_line != nline:
editor.markerDelete(
nline,
editor.DEACTIVE_BREAK_MARKER_NUM)
editor.markerAdd(
nline, editor.ACTIVE_BREAK_MARKER_NUM)
else:
editor.markerDelete(
nline,
editor.ARROW_DEACTIVE_BREAK_MARKER_NUM)
editor.markerAdd(
nline,
editor.ARROW_ACTIVE_BREAK_MARKER_NUM)
editor.line_click[nline] = 1
#reply = None
#self.command = None
if reply.command in DBG_ADVANCE_COMMANDS:
if reply.command == DBG_CMD_CONTINUE:
# delete current line marker
for m in self.editorManager.editors:
if self.current_debug_file == m:
# check what kind of arrow marker was on the current line and delete just arrow mark not break points
self.editorManager.tab.editors[
self.editorManager.editors[m]
[1]].markerDelete(
self.editorManager.tab.editors[
self.editorManager.editors[m]
[1]].current_line,
Editor.ARROW_MARKER_NUM)
self.editorManager.tab.editors[
self.editorManager.editors[m]
[1]].current_line = -1
# clean backtrace and debug windows
self.backtraceWindow.clearTraceTable(0)
self.debugWindow.clearLocalTable(0)
self.debugWindow.clearGlobalTable(0)
# Leave the debug UI disabled, and wait for the info command to return
self.send_debugger_command(DBG_CMD_INFO)
return foo
f = debugger_reply_finished(reply)
QObject.connect(reply, SIGNAL('finished()'), f)
reply.command = command[:]
if command in DBG_ADVANCE_COMMANDS:
# TODO: Here we should disable the debug UI
self.main.debug_stepinto.setEnabled(False)
self.main.debug_stepover.setEnabled(False)
self.main.debug_stepout.setEnabled(False)
self.main.debug_pause_bt.setEnabled(True)
self.main.debug_continue_bt.setEnabled(False)
self.main.ui.actionContinue.setEnabled(False)
self.main.ui.actionPause.setEnabled(True)
self.main.ui.actionStep_into.setEnabled(False)
self.main.ui.actionStep_over.setEnabled(False)
self.main.ui.actionStep_out.setEnabled(False)
self.main.debug_run = True
return True
def refresh():
req = nam.get(QNetworkRequest(QUrl("http://reestr.rublacklist.net/")))
req.finished.connect(lambda: handleFinished(req))
def _wrapRequest(self, request):
request = QNetworkRequest(request)
request.setAttribute(self._REQUEST_ID, self._next_id)
self._next_id += 1
return request
def _getRequest(self, url):
request = QNetworkRequest(url)
request.setRawHeader("Content-Type", "application/json")
request.setRawHeader('User-Agent', 'QGISCartoDB 0.2.x')
return request
def run(self):
xx = str(self.__xx)
yy = str(self.__yy)
if abs(float(xx)) > 180 or abs(float(yy)) > 90:
QgsMessageLog.logMessage(
self.tr('Worker: %s, %s are wrong coords!') % (xx, yy),
self.tr('OSMInfo'),
QgsMessageLog.INFO
)
self.gotError.emit(self.tr('Worker: %s, %s are wrong coords!') % (xx, yy))
return
url = 'http://overpass-api.de/api/interpreter'
request = QNetworkRequest(QUrl(url))
request.setHeader(QNetworkRequest.ContentTypeHeader, 'application/x-www-form-urlencoded');
qnam = QgsNetworkAccessManager.instance()
# around request
dist = PluginSettings.distance_value()
timeout = PluginSettings.timeout_value()
request_data = '[timeout:%s][out:json];(node(around:%s,%s,%s);way(around:%s,%s,%s);relation(around:%s,%s,%s););out tags geom;' % (timeout, dist, yy, xx, dist, yy, xx, dist, yy, xx)
reply1 = qnam.post(request, QByteArray(request_data))
loop = QEventLoop()
reply1.finished.connect(loop.quit)
loop.exec_()
if reply1.error() != QNetworkReply.NoError:
reply1.deleteLater()
self.gotError.emit(self.tr('Error getting data from the server'))
return
try:
data = reply1.readAll()
l1 = json.loads(str(data))['elements']
reply1.deleteLater()
except:
self.gotError.emit(self.tr('Error parsing data'))
return
finally:
reply1.deleteLater()
request_data = '[timeout:%s][out:json];is_in(%s,%s)->.a;way(pivot.a);out tags geom;relation(pivot.a);out geom;' % (timeout, yy, xx)
reply2 = qnam.post(request, QByteArray(request_data))
loop = QEventLoop()
reply2.finished.connect(loop.quit)
loop.exec_()
if reply2.error() != QNetworkReply.NoError:
reply2.deleteLater()
self.gotError.emit(self.tr('Error getting data from the server'))
return
try:
data = reply2.readAll()
l2 = json.loads(str(data))['elements']
except:
self.gotError.emit(self.tr('Error parsing data'))
return
finally:
reply2.deleteLater()
self.gotData.emit(l1, l2)
def _wrapRequest(self, request):
req = QNetworkRequest(request)
req.setAttribute(self._REQUEST_ID, next(self._request_ids))
if hasattr(request, 'timeout'):
req.timeout = request.timeout
return req
def getmk(self):
global manager
self.mkreq = QNetworkRequest(QUrl(self.mkurl))
self.mkreply = manager.get(self.mkreq)
QtCore.QObject.connect(self.mkreply, QtCore.SIGNAL("finished()"),
self.mkfinished)
class NetworkMixin(object):
"""A mixin that can be used to send network request and receive replies."""
request_url = str
error = None
results = None
def __init__(self, request_url, geojson=False):
self.request_url = request_url
self.manager = QgsNetworkAccessManager.instance()
self.reply = None
self.url = QUrl(self.request_url)
self.req = QNetworkRequest(self.url)
self.results = QByteArray()
self.auth_token = get_authtoken()
# Paginated data
self._pagination_exhausted = False
self.geojson = geojson
if geojson:
self.combine_new_data = self.combine_new_geojson_data
def cancel_request(self):
"""Abort the request."""
self.reply.abort()
def connect_request(self):
"""Process the request."""
LOGGER.info('Requesting "%s"', self.request_url)
self.reply.readyRead.connect(self.connection_read_data)
self.reply.finished.connect(self.connection_finished)
self.reply.error.connect(self.connection_error)
def connect_paginated_request(self, prev_data, callback):
"""Process the request of a paginated resource."""
LOGGER.info('Requesting "%s" (paginated)', self.request_url)
self.reply.readyRead.connect(self.connection_read_data)
self.reply.finished.connect(
functools.partial(self._handle_paginated_data, prev_data,
callback))
self.reply.error.connect(self.connection_error)
def connect_get(self):
"""Send get request."""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.reply = self.manager.get(self.req)
self.connect_request()
def connect_post(self, data):
"""Send post request.
:param data: Context data to use with template
:type data: QByteArray
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.reply = self.manager.post(self.req, data)
self.connect_request()
def connect_json_post(self, data):
"""Send post request with json string.
:param data: Json string data
:type data: str
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.req.setRawHeader("Content-Type", "application/json")
self.reply = self.manager.post(self.req, data)
self.connect_request()
def connect_json_put(self, data):
"""Send put request with json string.
:param data: Json string data
:type data: str
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.req.setRawHeader("Content-Type", "application/json")
json_string = QByteArray(data)
p_buffer = QBuffer(self.manager)
p_buffer.setData(json_string)
self.reply = self.manager.sendCustomRequest(self.req, 'PUT', p_buffer)
self.connect_request()
def connect_json_patch(self, data):
"""Send patch request with json string.
:param data: Json string data
:type data: str
"""
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.req.setRawHeader("Content-Type", "application/json")
json_string = QByteArray(data)
p_buffer = QBuffer(self.manager)
p_buffer.setData(json_string)
self.reply = self.manager.sendCustomRequest(self.req, 'PATCH',
p_buffer)
self.connect_request()
def connect_get_paginated(self, prev_data=None, callback=None):
if self.auth_token:
# Add authentication token to request
self.req.setRawHeader('Authorization',
'token %s' % self.auth_token)
self.reply = self.manager.get(self.req)
self.connect_paginated_request(
prev_data or self.results, callback
or self._set_results_and_complete)
def _handle_paginated_data(self, prev_data, callback):
"""
Once all data from single request is returned, merge it with
previous responses and follow pagination.
"""
# Add new request's data to prev_data
resp_txt = str(self.results)
merged_data = self.combine_new_data(str(prev_data or ''), resp_txt)
merged_data = QByteArray(merged_data)
# Get next page
next_url = None
try:
next_url = json.loads(resp_txt).get('next')
except ValueError:
pass
if next_url:
next_req = NetworkMixin(next_url, geojson=self.geojson)
return next_req.connect_get_paginated(prev_data=merged_data,
callback=callback)
LOGGER.debug('Pagination finished at "%s"', self.request_url)
callback(merged_data)
def _set_results_and_complete(self, results):
self.results = results
self._pagination_exhausted = True
self.connection_finished()
@staticmethod
def combine_new_data(old_data_str, new_data_str):
"""
Merge results from new page of data with results from previous
pages of data.
Expects a response in a format such as:
{
"count": 10,
"next": null,
"previous": null,
"results": [ ... ]
}
"""
old_data = ''
new_data = ''
try:
old_data = json.loads(old_data_str or '[]')
new_data = json.loads(new_data_str)
except ValueError:
pass
results_new_data = []
if 'results' in new_data:
results_new_data = new_data['results']
return json.dumps(old_data + results_new_data)
@staticmethod
def combine_new_geojson_data(old_data_str, new_data_str):
"""
Merge results from new page of GeoJSON data with results from
previous pages of data.
Expects a response in a format such as:
{
"count": 10,
"next": null,
"previous": null,
"results": {
"type": "FeatureCollection",
"features": [ ... ]
}
}
"""
old_data = dict()
new_data = dict()
try:
old_data = json.loads(old_data_str or '[]')
new_data = json.loads(new_data_str)
except ValueError:
pass
if not old_data and 'results' in new_data:
return json.dumps(new_data['results'])
if 'features' in old_data and 'results' in new_data:
old_data['features'] += new_data['results']['features']
return json.dumps(old_data)
def connection_read_data(self):
"""Get data from self.reply and append it to results."""
self.results += self.reply.readAll()
def get_json_results(self):
"""Convert results to json object."""
return json.loads(str(self.results))
def connection_error(self):
"""Handle error connection."""
error_result = self.reply.error
try:
http_code = int(
self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute))
except TypeError:
http_code = None
self.http_code = 404
if error_result == QNetworkReply.UnknownNetworkError:
msg = 'The network is unreachable.'
elif error_result == QNetworkReply.ProtocolUnknownError \
or error_result == QNetworkReply.HostNotFoundError:
msg = 'Host not found : %s' % self.url.encodedHost()
else:
if http_code:
msg = 'Error code:' + str(http_code)
self.http_code = http_code
else:
msg = 'Can\'t find the server'
LOGGER.debug(msg)
self.error = msg
@abc.abstractmethod
def connection_finished(self):
return
def is_finished(self, paginated=False):
return (self._pagination_exhausted
if paginated else self.reply.isFinished())
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None,
blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k, v)
# if self.authid:
# self.msg_log("Update request w/ authid: {0}".format(self.authid))
# QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, file):
body = body.read()
self.reply = func(req, body)
else:
self.reply = func(req)
# if self.authid:
# self.msg_log("Update reply w/ authid: {0}".format(self.authid))
# QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)
# necessary to trap local timout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(
self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
def request_download(manager, target_url):
url = QUrl(target_url)
request = QNetworkRequest(url)
print "Download start time: {}".format(datetime.now())
manager.get(request)
def request(self,
url,
user_agent='Mozilla',
cookies={},
timeout=15,
method='get',
data=None,
headers={}):
url_info = urlsplit(url)
self.resource_list = []
loop = QEventLoop()
self.view.loadFinished.connect(loop.quit)
# Timeout
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(loop.quit)
timer.start(timeout * 1000)
# User-Agent
self.page.user_agent = user_agent
# Cookies
cookie_obj_list = []
for name, value in cookies.items():
domain = ('.' + url_info.netloc).split(':')[0]
#print 'CREATE COOKIE %s=%s' % (name, value)
#print 'DOMAIN = %s' % domain
cookie_obj = QNetworkCookie(name, value)
cookie_obj.setDomain(domain)
cookie_obj_list.append(cookie_obj)
self.cookie_jar.setAllCookies(cookie_obj_list)
# Method
method_obj = getattr(QNetworkAccessManager,
'%sOperation' % method.capitalize())
# Ensure that Content-Type is correct if method is post
if method == 'post':
headers['Content-Type'] = 'application/x-www-form-urlencoded'
# Post data
if data is None:
data = QByteArray()
# Request object
request_obj = QNetworkRequest(QUrl(url))
# Headers
for name, value in headers.items():
request_obj.setRawHeader(name, value)
# Make a request
self.view.load(request_obj, method_obj, data)
loop.exec_()
if timer.isActive():
request_resource = None
url = str(self.page.mainFrame().url().toString()).rstrip('/')
for res in self.resource_list:
if url == res.url or url == res.url.rstrip('/'):
request_resource = res
break
if request_resource:
return self.build_response(request_resource)
else:
raise KitError('Request was successfull but it is not possible '\
'to associate the request to one of received responses')
else:
raise KitError('Timeout while loading %s' % url)
def create_request(self, operation, request, data):
""" Reimplemented to intercept requests. Stops blacklisted requests,
matches requests with replies. Stores on a PostgreSQL database.
"""
if es_url_local(request.url()) and not es_font(request.url()):
return QNetworkAccessManager.createRequest(self, operation,
request, data)
if (usando_whitelist(self.whitelist, request.url())
or es_font(request.url())):
print "FILTERING %s" % request.url().toString()[:255]
return QNetworkAccessManager.createRequest(
self, operation, QNetworkRequest(QUrl("about:blank")), data)
request.setAttribute(QNetworkRequest.HttpPipeliningAllowedAttribute,
True)
response = QNetworkAccessManager.createRequest(self, operation,
request, data)
if operation == QNetworkAccessManager.PostOperation:
post_str = unicode(data.peek(4096))
data_json = filtra(parse_qsl(post_str, keep_blank_values=True))
else:
data_json = None
def indice(reply, idx):
""" This function returns a closure enclosing the current index
and its associated reply. This is required since there's no
knowing when (if at all) a request will be replied to; once it
happens, the index will surely have changed.
The 'cheatgc' list is required because of a bug where the reference
for the 'reply' and 'idx' objects is lost sometimes after 'ret'
completes, despite having created a closure referencing them. By
appending to the list, garbage collection is prevented.
"""
# please don't do garbage collection...
self.cheatgc.append(reply)
self.cheatgc.append(idx)
def ret():
""" Closure; freezes 'reply' and 'idx' so they will be
accessible at the time the request finalizes and this closure
is called.
"""
try:
encabezados = filtra(reply.rawHeaderPairs())
(statuscode, _) = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute).toInt()
if self.log:
self.log.store_reply({
"id":
self.instance_id,
"idx":
idx,
"scheme":
unicode(reply.url().scheme()),
"host":
unicode(reply.url().host()),
"path":
unicode(reply.url().path()),
"query":
filtra(reply.url().encodedQueryItems()),
"fragment":
notnull(unicode(reply.url().fragment())),
"status":
statuscode,
"headers":
encabezados
})
# ...until we're done with the request
# (pyqt/sip related trouble)
self.cheatgc.remove(reply)
self.cheatgc.remove(idx)
#print "[%s]" % (len(self.cheatgc))
except NameError as name_error:
print name_error
print "Except NameError!"
return ret
response.finished.connect(indice(response, self.count))
if self.log:
headers = []
for header in request.rawHeaderList():
headers.append((header, request.rawHeader(header)))
self.log.store_request({
"id":
self.instance_id,
"idx":
self.count,
"op":
OPERATIONS[operation],
"scheme":
unicode(request.url().scheme()),
"host":
unicode(request.url().host()),
"path":
unicode(request.url().path()),
"query":
filtra(request.url().encodedQueryItems()),
"fragment":
notnull(unicode(request.url().fragment())),
"data":
data_json,
"source":
unicode(request.originatingObject().requestedUrl().host()),
"headers":
filtra(headers)
})
self.count += 1
return response
def executeRequest(self, url, callback, post=None):
self.messageBar.clearWidgets()
self.cmbOfferings.setEnabled(False)
self.tabWidget.setEnabled(False)
if post:
request = QNetworkRequest(url)
request.setRawHeader('Content-Type', 'application/xml')
self.reply = QgsNetworkAccessManager.instance().post(request, post)
else:
self.reply = QgsNetworkAccessManager.instance().get(
QNetworkRequest(url))
progressMessageBar = self.messageBar.createMessage(
self.tr("Please wait while downloading"))
progressBar = QtGui.QProgressBar(self)
progressBar.setMinimum(0)
progressBar.setFormat(self.tr("%v bytes downloaded!"))
progressBar.setAlignment(QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
def updateProgress(read, total):
try:
progressBar.setMaximum(total)
progressBar.setValue(read)
except:
pass
updateProgress(0, 0)
self.reply.downloadProgress.connect(updateProgress)
progressMessageBar.layout().addWidget(progressBar)
btnAbort = QtGui.QPushButton(self.tr("Abort"))
btnAbort.clicked.connect(self.reply.abort)
progressMessageBar.layout().addWidget(btnAbort)
fd, replyFilename = mkstemp(suffix=".xml",
prefix=callback.__name__,
dir=self.selectedWorkDir())
os.close(fd)
replyFile = open(replyFilename, "w")
def replyReadyRead():
try:
replyFile.write(self.reply.readAll())
except:
pass
self.reply.readyRead.connect(replyReadyRead)
def finishRequest():
replyFile.close()
self.cmbOfferings.setEnabled(True)
self.tabWidget.setEnabled(True)
self.messageBar.clearWidgets()
if self.reply.error() != QNetworkReply.NoError:
self.messageBar.pushMessage(self.reply.errorString(),
QgsMessageBar.CRITICAL)
else:
callback(replyFilename)
self.reply.deleteLater()
self.reply.finished.connect(finishRequest)
self.messageBar.pushWidget(progressMessageBar, QgsMessageBar.INFO)
def openUrl(self, url):
self.replayDownload.get(QNetworkRequest(url))
def download_url(manager, url, output_path, progress_dialog=None):
"""Download file from url.
:param manager: A QNetworkAccessManager instance
:type manager: QNetworkAccessManager
:param url: URL of file
:type url: str
:param output_path: Output path
:type output_path: str
:param progress_dialog: Progress dialog widget
:returns: True if success, otherwise returns a tuple with format like this
(QNetworkReply.NetworkError, error_message)
:raises: IOError - when cannot create output_path
"""
# prepare output path
myFile = QFile(output_path)
if not myFile.open(QFile.WriteOnly):
raise IOError(myFile.errorString())
# slot to write data to file
def write_data():
"""Write data to a file."""
myFile.write(myReply.readAll())
myRequest = QNetworkRequest(QUrl(url))
myReply = manager.get(myRequest)
myReply.readyRead.connect(write_data)
if progress_dialog:
# progress bar
def progress_event(received, total):
"""Update progress.
:param received: Data received so far.
:type received: int
:param total: Total expected data.
:type total: int
"""
# noinspection PyArgumentList
QCoreApplication.processEvents()
progress_dialog.setLabelText("%s / %s" % (received, total))
progress_dialog.setMaximum(total)
progress_dialog.setValue(received)
# cancel
def cancel_action():
"""Cancel download."""
myReply.abort()
myReply.downloadProgress.connect(progress_event)
progress_dialog.canceled.connect(cancel_action)
# wait until finished
while not myReply.isFinished():
# noinspection PyArgumentList
QCoreApplication.processEvents()
myFile.close()
myResult = myReply.error()
if myResult == QNetworkReply.NoError:
return True
else:
return myResult, str(myReply.errorString())
def insertFromMimeData(self, source):
""" Intended behavior
If copy/drag something that hasUrls, then check the extension name:
if image then apply image pattern 
else apply link pattern [text](http://example.net)
If copy/drag something that hasImage, then ask for file name
If copy/drag something that hasHtml, then html2text
Else use the default insertFromMimeData implementation
"""
item = self.parent.notesTree.currentItem()
attDir = self.parent.notesTree.itemToAttachmentDir(item)
if not QDir(attDir).exists():
QDir().mkpath(attDir)
if source.hasUrls():
for qurl in source.urls():
url = qurl.toString()
filename, extension = os.path.splitext(url)
filename = os.path.basename(filename)
newFilePath = os.path.join(attDir,
filename + extension).replace(
os.sep, '/')
relativeFilePath = newFilePath.replace(
self.settings.notebookPath, "..")
attachments = self.settings.attachmentImage + self.settings.attachmentDocument
if QUrl(qurl).isLocalFile():
if extension.lower() in attachments:
nurl = url.replace("file://", "")
QFile.copy(nurl, newFilePath)
self.parent.updateAttachmentView()
if extension.lower() in self.settings.attachmentImage:
text = "" % (filename, relativeFilePath)
elif extension.lower(
) in self.settings.attachmentDocument:
text = "[%s%s](%s)\n" % (filename, extension,
relativeFilePath)
else:
text = "[%s%s](%s)\n" % (filename, extension, url)
else:
if extension.lower() in attachments:
self.downloadAs = newFilePath
self.networkManager.get(QNetworkRequest(qurl))
if extension.lower() in self.settings.attachmentImage:
text = "" % (filename, relativeFilePath)
elif extension.lower(
) in self.settings.attachmentDocument:
text = "[%s%s](%s)\n" % (filename, extension,
relativeFilePath)
else:
text = "[%s%s](%s)\n" % (filename, extension, url)
super(MikiEdit,
self).insertFromMimeData(self.mimeFromText(text))
elif source.hasImage():
img = source.imageData()
attDir = self.parent.notesTree.itemToAttachmentDir(item)
dialog = LineEditDialog(attDir, self)
if dialog.exec_():
fileName = dialog.editor.text()
if not QFileInfo(fileName).suffix():
fileName += '.jpg'
filePath = os.path.join(attDir, fileName).replace(os.sep, '/')
img.save(filePath)
relativeFilePath = filePath.replace(self.settings.notebookPath,
"..")
text = "" % (fileName, relativeFilePath)
super(MikiEdit,
self).insertFromMimeData(self.mimeFromText(text))
elif source.hasHtml():
html = source.html()
if HAS_HTML2TEXT:
backToMarkdown = html2text.HTML2Text()
markdown = backToMarkdown.handle(html)
super(MikiEdit,
self).insertFromMimeData(self.mimeFromText(markdown))
else:
super(MikiEdit,
self).insertFromMimeData(self.mimeFromText(html))
else:
super(MikiEdit, self).insertFromMimeData(source)
def download(self):
"""Downloading the file.
:returns: True if success, otherwise returns a tuple with format like
this (QNetworkReply.NetworkError, error_message)
:raises: IOError - when cannot create output_path
"""
# Prepare output path
self.output_file = QFile(self.output_path)
if not self.output_file.open(QFile.WriteOnly):
raise IOError(self.output_file.errorString())
# Prepare downloaded buffer
self.downloaded_file_buffer = QByteArray()
# Request the url
request = QNetworkRequest(QUrl(self.url))
self.reply = self.manager.get(request)
self.reply.readyRead.connect(self.update_buffer_container)
self.reply.finished.connect(self.write_data)
if self.progress_dialog:
# progress bar
def progress_event(received, total):
"""Update progress.
:param received: Data received so far.
:type received: int
:param total: Total expected data.
:type total: int
"""
# noinspection PyArgumentList
QCoreApplication.processEvents()
label_text = "%s / %s" % (received, total)
self.progress_dialog.setLabelText(label_text)
self.progress_dialog.setMaximum(total)
self.progress_dialog.setValue(received)
# cancel
def cancel_action():
"""Cancel download."""
self.reply.abort()
self.reply.downloadProgress.connect(progress_event)
self.progress_dialog.canceled.connect(cancel_action)
# Wait until finished
# On Windows 32bit AND QGIS 2.2, self.reply.isFinished() always
# returns False even after finished slot is called. So, that's why we
# are adding self.finished_flag (see #864)
while not self.reply.isFinished() and not self.finished_flag:
# noinspection PyArgumentList
QCoreApplication.processEvents()
result = self.reply.error()
if result == QNetworkReply.NoError:
return True, None
else:
return result, str(self.reply.errorString())
# coding: utf-8
from PyQt4.QtCore import QUrl
from PyQt4.QtNetwork import QNetworkRequest, QSslError
from qgis.core import QgsNetworkAccessManager
from qgis.gui import QgsAuthSslErrorsDialog
url = 'http://fm4dd.com/openssl/source/PEM/certs/1024b-rsa-example-cert.pem'
networkAccessManager = QgsNetworkAccessManager.instance()
req = QNetworkRequest(QUrl(url))
reply = networkAccessManager.get(
req) # Return a QNetworkReply (http://doc.qt.io/qt-4.8/qnetworkreply.html)
auth_ssl_errors_dialog = QgsAuthSslErrorsDialog(
reply, [QSslError(QSslError.InvalidNotAfterField)])
auth_ssl_errors_dialog.show()
def onlineTreeDoubleClicked(self, item):
if hasattr(item, "url"):
self.replayDownload.get(QNetworkRequest(QtCore.QUrl(item.url)))
node.dispatchEvent(event);
});
}
};
var fix = document.querySelector('.gt-revert-correct-message a');
if (fix) {
fix.click();
setTimeout(listen, 1000 + Math.random() * 4000);
} else {
listen();
}
}, 1000 + Math.random() * 4000);
'''
FAKE = QNetworkRequest(QUrl(""))
class Google(Service):
"""
Provides a Service-compliant implementation for Google Translate.
"""
__slots__ = [
'_nam', # recycled QNetworkAccessManager instance for all requests
'_page', # recycled QWebPage instance for all requests
'_frame', # recycled QWebFrame instance for all requests
'_cb', # when a request is in-process, dict set to the callbacks
]
NAME = "Google Translate"
def download_url(manager, url, output_path, progress_dialog=None):
global xml_result
xml_result = []
# set up the output path
if output_path is not None:
out_file = QFile(output_path)
if not out_file.open(QFile.WriteOnly):
raise IOError(out_file.errorString())
# write data to file
def write_data():
global xml_result
xml_result = reply.readAll().data()
out_file.write(xml_result)
out_file.flush()
# read data from response
def read_data():
global xml_result
xml_result.append(reply.readAll().data())
# request the content of the url
request = QNetworkRequest(QUrl(url))
reply = manager.get(request)
if output_path is None:
reply.readyRead.connect(read_data)
else:
reply.readyRead.connect(write_data)
if progress_dialog:
def progress_event(received, total):
QCoreApplication.processEvents()
progress_dialog.setLabelText("%s / %s" % (received, total))
progress_dialog.setMaximum(total)
progress_dialog.setValue(received)
# cancel the download
def cancel_action():
reply.abort()
reply.downloadProgress.connect(progress_event)
progress_dialog.canceled.connect(cancel_action)
# wait until donwload is finished
while not reply.isFinished():
QCoreApplication.processEvents()
result = reply.error()
if result == QNetworkReply.NoError:
if output_path is None:
return True, None, xml_result
else:
out_file.close()
return True, None, xml_result
else:
if output_path is not None:
out_file.close()
return result, str(reply.errorString())
if progress_dialog:
progress_dialog.close()
def get(self):
request = QNetworkRequest(self.url)
request.setRawHeader('User-Agent', osDetector.getOs() + " - UDS Connector " + VERSION)
self._manager.get(request)
def get(self, url):
future = Future()
url = QUrl(url)
request = QNetworkRequest(url)
request.setRawHeader(b"User-Agent", b"OWImageViewer/1.0")
request.setAttribute(
QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache
)
# Future yielding a QNetworkReply when finished.
reply = self._netmanager.get(request)
future._reply = reply
@future.add_done_callback
def abort_on_cancel(f):
# abort the network request on future.cancel()
if f.cancelled() and f._reply is not None:
f._reply.abort()
n_redir = 0
def on_reply_ready(reply, future):
nonlocal n_redir
# schedule deferred delete to ensure the reply is closed
# otherwise we will leak file/socket descriptors
reply.deleteLater()
future._reply = None
if reply.error() == QNetworkReply.OperationCanceledError:
# The network request was cancelled
reply.close()
future.cancel()
return
if reply.error() != QNetworkReply.NoError:
# XXX Maybe convert the error into standard
# http and urllib exceptions.
future.set_exception(Exception(reply.errorString()))
reply.close()
return
# Handle a possible redirection
location = reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if location is not None and n_redir < 1:
n_redir += 1
location = reply.url().resolved(location)
# Retry the original request with a new url.
request = QNetworkRequest(reply.request())
request.setUrl(location)
newreply = self._netmanager.get(request)
future._reply = newreply
newreply.finished.connect(
partial(on_reply_ready, newreply, future))
reply.close()
return
reader = QImageReader(reply)
image = reader.read()
reply.close()
if image.isNull():
future.set_exception(Exception(reader.errorString()))
else:
future.set_result(image)
reply.finished.connect(partial(on_reply_ready, reply, future))
return future
def load_url_async(url):
m = QgsNetworkAccessManager.instance()
req = QNetworkRequest(QUrl(url))
req.setRawHeader('User-Agent', 'Magic Browser')
reply = m.get(req)
return reply
def initGui(self):
request = QNetworkRequest(QUrl(self.SEARCH_URL))
self.networkManager.get(request)
def createNetworkReply(self, uri):
url = QUrl(uri, QUrl.TolerantMode)
request = QNetworkRequest(url)
reply = self.network.get(request)
self.networkReplies.append(reply)
return reply
socket.disconnectFromHost()
if proxy_socket.error() != QTcpSocket.RemoteHostClosedError:
url = proxy_socket.property('url').toUrl()
error_string = proxy_socket.errorString()
if self.debug:
self.log.write('Error for %s %s\n\n' % (url, error_string))
proxy_socket.deleteLater()
if __name__ == '__main__':
app = QCoreApplication(sys.argv)
server = HTTPProxy()
manager = QNetworkAccessManager()
manager.finished.connect(server.stopServing)
manager.finished.connect(app.quit)
proxy = QNetworkProxy()
proxy.setType(QNetworkProxy.HttpProxy)
proxy.setHostName('127.0.0.1')
proxy.setPort(server.port())
proxy.setUser(server.username)
proxy.setPassword(server.password)
manager.setProxy(proxy)
reply = manager.get(QNetworkRequest(QUrl('http://aws.amazon.com/')))
sys.exit(app.exec_())