def testInvalidAuthFileDownload(self):
"""
Download a protected map tile without authcfg
"""
qs = "?" + "&".join(["%s=%s" % i for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, None, False)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
self.assertTrue(self.error_was_called)
self.assertTrue("Download failed: Host requires authentication" in str(self.error_args), "Error args is: %s" % str(self.error_args))
def wait_object_destruction(self, my_object):
loop = QEventLoop()
name = my_object.objectName()
my_object.destroyed.connect(loop.quit)
loop.exec_()
self.assertIsNone(find_window(name))
return None
def __sync_request(self, url):
_url = QUrl(url)
_request = QNetworkRequest(_url)
self.__replies.append(_request)
QgsNetworkAccessManager.instance().sslErrors.connect(self.__supress_ssl_errors)
_reply = QgsNetworkAccessManager.instance().get(_request)
# wait
loop = QEventLoop()
_reply.finished.connect(loop.quit)
loop.exec_()
_reply.finished.disconnect(loop.quit)
QgsNetworkAccessManager.instance().sslErrors.disconnect(self.__supress_ssl_errors)
loop = None
error = _reply.error()
if error != QNetworkReply.NoError:
raise Exception(error)
result_code = _reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
result = _reply.readAll()
self.__replies.append(_reply)
_reply.deleteLater()
if result_code in [301, 302, 307]:
redirect_url = _reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
return self.__sync_request(redirect_url)
else:
return result
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(
self._url, self.result_path, delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
else:
return self.result_path
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
# query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('q', query)
query_string.addQueryItem('format', 'json')
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def fetchFiles(self, urls):
self.logT("TileLayer.fetchFiles() starts")
# create a QEventLoop object that belongs to the current thread (if ver. > 2.1, it is render thread)
eventLoop = QEventLoop()
self.logT("Create event loop: " + str(eventLoop)) # DEBUG
# QObject.connect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
self.allRepliesFinished.connect(eventLoop.quit)
# create a timer to watch whether rendering is stopped
watchTimer = QTimer()
watchTimer.timeout.connect(eventLoop.quit)
# send a fetch request to the main thread
# self.emit(SIGNAL("fetchRequest(QStringList)"), urls)
self.fetchRequest.emit(urls)
# wait for the fetch to finish
tick = 0
interval = 500
timeoutTick = self.downloadTimeout / interval
watchTimer.start(interval)
while tick < timeoutTick:
# run event loop for 0.5 seconds at maximum
eventLoop.exec_()
if debug_mode:
qDebug("watchTimerTick: %d" % tick)
qDebug("unfinished downloads: %d" % self.downloader.unfinishedCount())
if self.downloader.unfinishedCount() == 0 or self.renderContext.renderingStopped():
break
tick += 1
watchTimer.stop()
if tick == timeoutTick and self.downloader.unfinishedCount() > 0:
self.log("fetchFiles timeout")
# self.emitShowBarMessage("fetchFiles timeout", duration=5) #DEBUG
self.downloader.abort()
self.downloader.errorStatus = Downloader.TIMEOUT_ERROR
files = self.downloader.fetchedFiles
watchTimer.timeout.disconnect(eventLoop.quit) #
# QObject.disconnect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
self.allRepliesFinished.disconnect(eventLoop.quit)
self.logT("TileLayer.fetchFiles() ends")
return files
def run(self):
ili2db_bin = get_ili2db_bin(self.tool, self.stdout, self.stderr)
if not ili2db_bin:
return
ili2db_jar_arg = ["-jar", ili2db_bin]
self.configuration.tool = self.tool
db_simple_factory = DbSimpleFactory()
db_factory = db_simple_factory.create_factory(self.tool)
config_manager = db_factory.get_db_command_config_manager(self.configuration)
args = config_manager.get_ili2db_args(False)
args_hide_password = config_manager.get_ili2db_args(True)
java_path = get_java_path(self.configuration.base_configuration)
proc = QProcess()
proc.readyReadStandardError.connect(
functools.partial(self.stderr_ready, proc=proc))
proc.readyReadStandardOutput.connect(
functools.partial(self.stdout_ready, proc=proc))
proc.start(java_path, ili2db_jar_arg + args)
if not proc.waitForStarted():
proc = None
if not proc:
raise JavaNotFoundError()
safe_args = ili2db_jar_arg + args_hide_password
safe_command = java_path + ' ' + ' '.join(safe_args)
self.process_started.emit(safe_command)
self.__result = Exporter.ERROR
loop = QEventLoop()
proc.finished.connect(loop.exit)
loop.exec()
self.process_finished.emit(proc.exitCode(), self.__result)
return self.__result
def download_image(self, url):
res = False
img = None
msg = {'text': '', 'level': Qgis.Warning}
if url:
self.logger.info(__name__, "Downloading file from {}".format(url))
msg_status_bar = "Downloading image from document repository (this might take a while)..."
with ProcessWithStatus(msg_status_bar):
if is_connected(TEST_SERVER):
nam = QNetworkAccessManager()
request = QNetworkRequest(QUrl(url))
reply = nam.get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
status = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if status == 200:
res = True
img = reply.readAll()
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to the server. The server might be down or the service cannot be reached at the given URL."
)
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to Internet.")
else:
res = False
msg['text'] = QCoreApplication.translate("SettingsDialog",
"Not valid URL")
if not res:
self.logger.log_message(__name__, msg['text'], msg['level'])
return res, img
def testInvalidAuthFileDownload(self):
"""
Download a protected map tile without authcfg
"""
qs = "?" + "&".join([
"%s=%s" % i
for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())
])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, None, False)
downloader.downloadCompleted.connect(
partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot,
'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot,
'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
self.assertTrue(self.error_was_called)
self.assertTrue(
"Download failed: Host requires authentication"
in str(self.error_args),
"Error args is: %s" % str(self.error_args))
def test_api_usage_async(self):
"""Test standard async API usage"""
api = NzElectoralApi('http://localhost:%s' % self.port)
nam = api.status()
self.last_result = ''
expected = {
"version": self.API_VERSION,
"gmsVersion": "LINZ_Output_20180108_2018_V1_00"
}
el = QEventLoop()
def f(nam):
"""Wrapper"""
self.last_result = api.parse_async(nam)['content']
nam.reply.finished.connect(partial(f, nam))
nam.reply.finished.connect(el.quit)
el.exec_(QEventLoop.ExcludeUserInputEvents)
self.assertEqual(self.last_result, expected)
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language", bytes(settings.value("default_language", "fr"), "utf8"))
req.setRawHeader(b"User-Agent", bytes(settings.value('http_user_agent', plugin_name()), "utf8"))
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def testStoreWithoutFileName(self):
"""
Test file storing and fetching
"""
f = self.getNewFile(b"New content")
# store
storedContent = self.storage.store(f.name, self.url + "/",
self.auth_config.id())
self.assertTrue(storedContent)
self.assertEqual(storedContent.status(), Qgis.ContentStatus.NotStarted)
spyErrorOccurred = QSignalSpy(storedContent.errorOccurred)
spyProgressChanged = QSignalSpy(storedContent.progressChanged)
loop = QEventLoop()
storedContent.stored.connect(loop.quit)
storedContent.errorOccurred.connect(loop.quit)
QTimer.singleShot(1, lambda: storedContent.store())
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertFalse(storedContent.errorString())
self.assertEqual(storedContent.url(),
self.url + "/" + os.path.basename(f.name))
self.assertEqual(storedContent.status(), Qgis.ContentStatus.Finished)
self.assertTrue(len(spyProgressChanged) > 0)
self.assertEqual(spyProgressChanged[-1][0], 100)
# fetch
fetchedContent = self.storage.fetch(
self.url + "/" + os.path.basename(f.name), self.auth_config.id())
self.assertTrue(fetchedContent)
self.assertEqual(fetchedContent.status(),
Qgis.ContentStatus.NotStarted)
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.fetched.connect(loop.quit)
fetchedContent.errorOccurred.connect(loop.quit)
QTimer.singleShot(1, lambda: fetchedContent.fetch())
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
self.assertFalse(fetchedContent.errorString())
self.assertTrue(fetchedContent.filePath())
self.checkContent(fetchedContent.filePath(), b"New content")
self.assertEqual(
os.path.splitext(fetchedContent.filePath())[1], '.txt')
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language", b"fr")
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
raise OverpassTimeoutException
# The download is done, checking for not complete OSM file.
# Overpass might aborted the request with HTTP 200.
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = (
'<remark> runtime error: Query timed out in "[a-z]+" at line '
'[\d]+ after ([\d]+) seconds. </remark>')
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
# Everything went fine
return self.result_path
def get_available_providers(providers_uri):
"""Fetch the list of available providers from BCS endpoint,
apparently this API method does not require auth"""
# For testing purposes, we can also access to a json file directly
if not providers_uri.startswith('http'):
try:
j = json.load(open(providers_uri, encoding='utf-8'))
except:
j = json.load(open(providers_uri))
else:
t = mktemp()
q = QgsFileDownloader(QUrl(providers_uri), t)
loop = QEventLoop()
q.downloadExited.connect(loop.quit)
loop.exec_()
if not os.path.isfile(t):
return []
with open(t) as f:
j = json.load(f)
os.unlink(t)
return j
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.queue = []
self.redirected_urls = {}
self.requestingUrls = []
self.replies = []
self.eventLoop = QEventLoop()
self.sync = False
self.fetchedFiles = {}
self.clearCounts()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.fetchTimedOut)
# network settings
self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
self.max_connection = 4
self.default_cache_expiration = 24
self.errorStatus = Downloader.NO_ERROR
def _get_sync(self, qurl: QUrl, timeout: int = 20000) -> Reply:
'''
synchronous GET-request
'''
request = QNetworkRequest(qurl)
## newer versions of QGIS (3.6+) support synchronous requests
#if hasattr(self._manager, 'blockingGet'):
#reply = self._manager.blockingGet(request, forceRefresh=True)
## use blocking event loop for older versions
#else:
loop = QEventLoop()
timer = QTimer()
timer.setSingleShot(True)
# reply or timeout break event loop, whoever comes first
timer.timeout.connect(loop.quit)
reply = self._manager.get(request)
reply.finished.connect(loop.quit)
timer.start(timeout)
# start blocking loop
loop.exec()
loop.deleteLater()
if not timer.isActive():
reply.deleteLater()
raise ConnectionError('Timeout')
timer.stop()
#if reply.error():
#self.error.emit(reply.errorString())
#raise ConnectionError(reply.errorString())
res = Reply(reply)
self.finished.emit(res)
return res
class OpenlayersRenderer(QgsMapLayerRenderer):
def __init__(self, layer, context, webPage, layerType):
""" Initialize the object. This function is still run in the GUI thread.
Should refrain from doing any heavy work.
"""
QgsMapLayerRenderer.__init__(self, layer.id())
self.context = context
self.controller = OpenlayersController(None,
context, webPage, layerType)
self.loop = None
def render(self):
""" do the rendering. This function is called in the worker thread """
debug("[WORKER THREAD] Calling request() asynchronously", 3)
QMetaObject.invokeMethod(self.controller, "request")
# setup a timer that checks whether the rendering has not been stopped
# in the meanwhile
timer = QTimer()
timer.setInterval(50)
timer.timeout.connect(self.onTimeout)
timer.start()
debug("[WORKER THREAD] Waiting for the async request to complete", 3)
self.loop = QEventLoop()
self.controller.finished.connect(self.loop.exit)
self.loop.exec_()
debug("[WORKER THREAD] Async request finished", 3)
painter = self.context.painter()
painter.drawImage(0, 0, self.controller.img)
return True
def onTimeout(self):
""" periodically check whether the rendering should not be stopped """
if self.context.renderingStopped():
debug("[WORKER THREAD] Cancelling rendering", 3)
self.loop.exit()
class OpenlayersRenderer(QgsMapLayerRenderer):
def __init__(self, layer, context, webPage, layerType):
""" Initialize the object. This function is still run in the GUI thread.
Should refrain from doing any heavy work.
"""
QgsMapLayerRenderer.__init__(self, layer.id())
self.context = context
self.controller = OpenlayersController(None, context, webPage,
layerType)
self.loop = None
def render(self):
""" do the rendering. This function is called in the worker thread """
debug("[WORKER THREAD] Calling request() asynchronously", 3)
QMetaObject.invokeMethod(self.controller, "request")
# setup a timer that checks whether the rendering has not been stopped
# in the meanwhile
timer = QTimer()
timer.setInterval(50)
timer.timeout.connect(self.onTimeout)
timer.start()
debug("[WORKER THREAD] Waiting for the async request to complete", 3)
self.loop = QEventLoop()
self.controller.finished.connect(self.loop.exit)
self.loop.exec_()
debug("[WORKER THREAD] Async request finished", 3)
painter = self.context.painter()
painter.drawImage(0, 0, self.controller.img)
return True
def onTimeout(self):
""" periodically check whether the rendering should not be stopped """
if self.context.renderingStopped():
debug("[WORKER THREAD] Cancelling rendering", 3)
self.loop.exit()
def process_delete_call(self, url, report_url=True):
"""
Run a DELETE request and return reply data
:param url: url for request
:param report_url: True if URL should be reported to feedback
:return: response or error message in json format
"""
if self.connection.read_only:
return {
"error": {
"msg": "Graphium connection is set to read-only!"
}
}
url_query = QUrl(url)
if report_url:
self.report_info('DELETE ' + url_query.toString())
request = QNetworkRequest(url_query)
if self.connection.auth_cfg != '':
self.auth = 0
config = QgsAuthMethodConfig()
QgsApplication.authManager().loadAuthenticationConfig(
self.connection.auth_cfg, config, True)
concatenated = config.configMap(
)['username'] + ":" + config.configMap()['password']
data = base64.encodebytes(concatenated.encode("utf-8")).replace(
'\n'.encode("utf-8"), ''.encode("utf-8"))
request.setRawHeader("Authorization".encode("utf-8"),
("Basic %s" % data).encode("utf-8"))
request.setRawHeader("Accept".encode("utf-8"),
"*/*".encode("utf-8"))
loop = QEventLoop()
reply = self.network_access_manager.deleteResource(request)
reply.finished.connect(loop.quit)
loop.exec_()
return self.process_q_reply(reply)
def testPythonCreateChildrenCalledFromCplusplus(self):
""" test createChildren() method implemented in Python, called from C++ """
loop = QEventLoop()
NUM_ITERS = 10 # set more to detect memory leaks
for i in range(NUM_ITERS):
tabSetDestroyedFlag = [False]
item = PyQgsDataConnectionItem(None, "name", "", "my_provider")
item.tabSetDestroyedFlag = tabSetDestroyedFlag
# Causes PyQgsDataConnectionItem.createChildren() to be called
item.populate()
# wait for populate() to have done its job
item.stateChanged.connect(loop.quit)
loop.exec_()
# Python object PyQgsLayerItem should still be alive
self.assertFalse(tabSetDestroyedFlag[0])
self.assertEqual(len(item.children()), 2)
self.assertEqual(item.children()[0].name(), "name")
self.assertEqual(item.children()[1].name(), "name2")
# Delete the object and make sure all deferred deletions are processed
item.destroyed.connect(loop.quit)
item.deleteLater()
loop.exec_()
# Check that the PyQgsLayerItem Python object is now destroyed
self.assertTrue(tabSetDestroyedFlag[0])
tabSetDestroyedFlag[0] = False
def _make_download(self, url, destination, cancel=False):
self.completed_was_called = False
self.error_was_called = False
self.canceled_was_called = False
self.progress_was_called = False
self.exited_was_called = False
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
if cancel:
downloader.downloadProgress.connect(downloader.cancelDownload)
loop.exec_()
def _make_download(self, url, destination, cancel=False):
self.completed_was_called = False
self.error_was_called = False
self.canceled_was_called = False
self.progress_was_called = False
self.exited_was_called = False
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
if cancel:
downloader.downloadProgress.connect(downloader.cancelDownload)
loop.exec_()
def testValidAuthFileDownload(self):
"""
Download a map tile with valid authcfg
"""
qs = "?" + "&".join(["%s=%s" % i for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, self.auth_config.id(), False)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
# Check the we've got a likely PNG image
self.assertTrue(self.completed_was_called)
self.assertTrue(os.path.getsize(destination) > 2000, "Image size: %s" % os.path.getsize(destination)) # > 1MB
with open(destination, 'rb') as f:
self.assertTrue(b'PNG' in f.read()) # is a PNG
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
for message in self.errors:
self.is_query_timed_out(message)
self.is_bad_request(message)
LOGGER.error(message)
if len(self.errors):
raise NetWorkErrorException('Overpass API', ', '.join(self.errors))
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
# Do not raise a QuickOSM exception here
# It must be a bug from QuickOSM
raise FileNotFoundError
self.check_file(self.result_path)
# Everything went fine
return self.result_path
def loadModel(self):
imd = None
try:
loader = ModelLoader(self.ui.mDataLineEdit.text())
models = loader.detect_models()
model_names = map(lambda m: m.name, models)
self._log_output("Looking up models: " + ', '.join(model_names))
ili = loader.gen_lookup_ili()
qDebug(ili)
wpsreq = self._create_wps_request(ili)
url = self.ui.mIlisMetaUrlLineEdit.text()
req = QNetworkRequest(QUrl(url))
req.setHeader(QNetworkRequest.ContentTypeHeader, 'application/xml')
reply = QgsNetworkAccessManager.instance().post(req, wpsreq)
# Wait for reply or timeout
loop = QEventLoop()
reply.finished.connect(loop.quit)
QTimer.singleShot(15000, reply.abort)
loop.exec_()
if reply.isFinished() and reply.error() == QNetworkReply.NoError:
result = reply.readAll()
imd = self._parse_wps_response(result)
except:
qDebug("Exception during IlisModel download")
if imd is None:
self._show_log_window()
QgsApplication.messageLog().logMessage(
"Couldn't download Ilismeta model", "Interlis",
Qgis.MessageLevel(1))
self.ui.mModelLineEdit.setText("")
else:
fh, imdfn = tempfile.mkstemp(suffix='.imd')
os.close(fh)
with codecs.open(imdfn, "w", encoding='utf-8') as file:
file.write(imd)
self.ui.mModelLineEdit.setText(imdfn)
def testValidAuthFileDownload(self):
"""
Download a map tile with valid authcfg
"""
qs = "?" + "&".join(["%s=%s" % i for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, self.auth_config.id(), False)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
# Check the we've got a likely PNG image
self.assertTrue(self.completed_was_called)
self.assertTrue(os.path.getsize(destination) > 700000, "Image size: %s" % os.path.getsize(destination)) # > 1MB
with open(destination, 'rb') as f:
self.assertTrue(b'PNG' in f.read()) # is a PNG
def render(self):
""" do the rendering. This function is called in the worker thread """
debug("[WORKER THREAD] Calling request() asynchronously", 3)
QMetaObject.invokeMethod(self.controller, "request")
# setup a timer that checks whether the rendering has not been stopped
# in the meanwhile
timer = QTimer()
timer.setInterval(50)
timer.timeout.connect(self.onTimeout)
timer.start()
debug("[WORKER THREAD] Waiting for the async request to complete", 3)
self.loop = QEventLoop()
self.controller.finished.connect(self.loop.exit)
self.loop.exec_()
debug("[WORKER THREAD] Async request finished", 3)
painter = self.context.painter()
painter.drawImage(0, 0, self.controller.img)
return True
def testPythonCreateChildrenCalledFromCplusplus(self):
"""
test createChildren() method implemented in Python, called from C++
This test was originally working under Qt 5.12, but is broken on newer Qt or sip
versions. The test currently segfaults, as the children created by the python QgsDataCollectionItem
subclass PyQgsDataConnectionItem are immediately garbage collected.
The SIP SIP_VIRTUAL_CATCHER_CODE in qgsdataitem.h is supposed to fix this situation by
adding an extra reference to the returned python objects, but the lines
// pyItem is given an extra reference which is removed when the C++ instance’s destructor is called.
sipTransferTo( pyItem, Py_None );
no longer have any effect and the object is still immediately deleted.
Attempted solutions include:
- all combinations of the existing VirtualCatcherCode with the different Factory/TransferBack annotations
- removing the VirtualCatcherCode and replacing with raw Factory/TransferBack annotations
- disabling the python garbage collection of the object entirely with sipTransferTo( pyItem, NULL )
When fixed this test should be moved back to test_qgsdataitem.py
"""
loop = QEventLoop()
NUM_ITERS = 10 # set more to detect memory leaks
for i in range(NUM_ITERS):
tabSetDestroyedFlag = [False]
item = PyQgsDataConnectionItem(None, "name", "", "my_provider")
item.tabSetDestroyedFlag = tabSetDestroyedFlag
# Causes PyQgsDataConnectionItem.createChildren() to be called
item.populate()
# wait for populate() to have done its job
item.stateChanged.connect(loop.quit)
loop.exec_()
# Python object PyQgsLayerItem should still be alive
self.assertFalse(tabSetDestroyedFlag[0])
children = item.children()
self.assertEqual(len(children), 2)
self.assertEqual(children[0].name(), "name")
self.assertEqual(children[1].name(), "name2")
del children
# Delete the object and make sure all deferred deletions are processed
item.destroyed.connect(loop.quit)
item.deleteLater()
loop.exec_()
# Check that the PyQgsLayerItem Python object is now destroyed
self.assertTrue(tabSetDestroyedFlag[0])
tabSetDestroyedFlag[0] = False
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language",
bytes(settings.value("default_language", "fr"), "utf8"))
req.setRawHeader(
b"User-Agent",
bytes(settings.value("http_user_agent", plugin_name()), "utf8"))
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def update_referecedata_cache_model(self, filter_models, type):
# updates the model and waits for the end
loop = QEventLoop()
self.ilireferencedatacache.model_refreshed.connect(lambda: loop.quit())
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(lambda: loop.quit())
timer.start(10000)
self.refresh_referencedata_cache(filter_models, type)
loop.exec()
return self.ilireferencedatacache.model
def render(self):
""" do the rendering. This function is called in the worker thread """
debug("[WORKER THREAD] Calling request() asynchronously", 3)
QMetaObject.invokeMethod(self.controller, "request")
# setup a timer that checks whether the rendering has not been stopped
# in the meanwhile
timer = QTimer()
timer.setInterval(50)
timer.timeout.connect(self.onTimeout)
timer.start()
debug("[WORKER THREAD] Waiting for the async request to complete", 3)
self.loop = QEventLoop()
self.controller.finished.connect(self.loop.exit)
self.loop.exec_()
debug("[WORKER THREAD] Async request finished", 3)
painter = self.context.painter()
painter.drawImage(0, 0, self.controller.img)
return True
def _post_sync(self,
qurl: QUrl,
timeout: int = 20000,
data: bytes = b'',
content_type=None):
'''
synchronous POST-request
'''
request = QNetworkRequest(qurl)
if content_type:
request.setHeader(QNetworkRequest.ContentTypeHeader, content_type)
# newer versions of QGIS (3.6+) support synchronous requests
if hasattr(self._manager, 'blockingPost'):
reply = self._manager.blockingPost(request,
data,
forceRefresh=True)
# use blocking event loop for older versions
else:
loop = QEventLoop()
timer = QTimer()
timer.setSingleShot(True)
# reply or timeout break event loop, whoever comes first
timer.timeout.connect(loop.quit)
reply = self._manager.post(request, data)
reply.finished.connect(loop.quit)
timer.start(timeout)
# start blocking loop
loop.exec()
loop.deleteLater()
if not timer.isActive():
reply.deleteLater()
raise ConnectionError('Timeout')
timer.stop()
if reply.error():
self.error.emit(reply.errorString())
raise ConnectionError(reply.errorString())
res = Reply(reply)
self.finished.emit(res)
return res
def get_topping_file_model(self, id_list):
topping_file_cache = IliToppingFileCache(
self.import_schema_configuration.base_configuration, id_list)
# we wait for the download or we timeout after 30 seconds and we apply what we have
loop = QEventLoop()
topping_file_cache.download_finished.connect(lambda: loop.quit())
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(lambda: loop.quit())
timer.start(30000)
topping_file_cache.refresh()
self.log_panel.print_info(self.tr("- - Downloading…"),
LogColor.COLOR_TOPPING)
if len(topping_file_cache.downloaded_files) != len(id_list):
loop.exec()
if len(topping_file_cache.downloaded_files) == len(id_list):
self.log_panel.print_info(
self.tr("- - All topping files successfully downloaded"),
LogColor.COLOR_TOPPING,
)
else:
missing_file_ids = id_list
for downloaded_file_id in topping_file_cache.downloaded_files:
if downloaded_file_id in missing_file_ids:
missing_file_ids.remove(downloaded_file_id)
self.log_panel.print_info(
self.
tr("- - Some topping files where not successfully downloaded: {}"
).format(" ".join(missing_file_ids)),
LogColor.COLOR_TOPPING,
)
return topping_file_cache.model
class NetworkAccessManager(object):
"""
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.
The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.
Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class
Usage 1 (blocking mode)
-----
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException as e:
# Handle exception
pass
Usage 2 (Non blocking mode)
-------------------------
::
NOTE! if blocking mode returns immediatly
it's up to the caller to manage listeners in case
of non blocking mode
nam = NetworkAccessManager(authcgf)
try:
nam.request('http://www.example.com', blocking=False)
nam.reply.finished.connect(a_signal_listener)
except RequestsException as e:
# Handle exception
pass
Get response using method:
nam.httpResult() that return a dictionary with keys:
'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
'content' - bytearray returned from reply
'ok' - request success [True, False]
'headers' - Dicionary containing the reply header
'reason' - fomatted message string with reply.errorString()
'exception' - the exception returne dduring execution
"""
def __init__(self,
authid=None,
disable_ssl_certificate_validation=False,
exception_class=None,
debug=True,
timeout=None):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.exception_class = exception_class
self.on_abort = False
self.blocking_mode = False
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'content': '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
if timeout is None:
timeout = 60
self.timeout = timeout
def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "NetworkAccessManager")
def httpResult(self):
return self.http_call_result
def auth_manager(self):
return QgsApplication.authManager()
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None,
blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
if k and v:
req.setRawHeader(k.encode(), v.encode())
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, io.IOBase):
body = body.read()
if isinstance(body, str):
body = body.encode()
if isinstance(body, dict):
body = str(json.dumps(body)).encode(encoding='utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkReply(self.reply, self.authid)
QgsNetworkAccessManager.instance().setTimeout(int(self.timeout) * 1000)
# necessary to trap local timeout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(
self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
#self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
pass
def requestTimedOut(self, reply):
"""Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
# adapt http_call_result basing on receiving qgs timer timout signal
self.exception_class = RequestsExceptionTimeout
self.http_call_result.exception = RequestsExceptionTimeout(
"Timeout error")
def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(
QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(
k.data(), encoding='utf-8')] = str(v.data(), encoding='utf-8')
self.http_call_result.headers[str(
k.data(), encoding='utf-8').lower()] = str(v.data(),
encoding='utf-8')
if err != QNetworkReply.NoError:
# handle error
# check if errorString is empty, if so, then set err string as
# reply dump
if re.match('(.)*server replied: $', self.reply.errorString()):
errString = self.reply.errorString(
) + self.http_call_result.content
else:
errString = self.reply.errorString()
# check if self.http_call_result.status_code is available (client abort
# does not produce http.status_code)
if self.http_call_result.status_code:
msg = "Network error #{0}: {1}".format(
self.http_call_result.status_code, errString)
else:
msg = "Network error: {0}".format(errString)
self.http_call_result.reason = msg
self.http_call_result.text = str(self.reply.readAll().data(),
encoding='utf-8')
self.http_call_result.ok = False
self.msg_log(msg)
# set return exception
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)
elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(
msg)
elif err == QNetworkReply.OperationCanceledError:
# request abort by calling NAM.abort() => cancelled by the user
if self.on_abort:
self.http_call_result.exception = RequestsExceptionUserAbort(
msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
self.http_call_result.exception = RequestsException(msg)
# overload exception to the custom exception if available
if self.exception_class:
self.http_call_result.exception = self.exception_class(msg)
else:
# Handle redirections
redirectionUrl = self.reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if redirectionUrl is not None and redirectionUrl != self.reply.url(
):
if redirectionUrl.isRelative():
redirectionUrl = self.reply.url().resolved(redirectionUrl)
msg = "Redirected from '{}' to '{}'".format(
self.reply.url().toString(), redirectionUrl.toString())
self.msg_log(msg)
self.reply.deleteLater()
self.reply = None
self.request(redirectionUrl.toString())
# really end request
else:
msg = "Network success #{0}".format(self.reply.error())
self.http_call_result.reason = msg
self.msg_log(msg)
ba = self.reply.readAll()
self.http_call_result.content = bytes(ba)
self.http_call_result.text = str(ba.data(), encoding='utf-8')
self.http_call_result.ok = True
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
for k, v in list(self.http_call_result.headers.items()):
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.content) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")
# clean reply
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
# Disconnect all slots
self.reply.sslErrors.disconnect(self.sslErrors)
self.reply.finished.disconnect(self.replyFinished)
self.reply.downloadProgress.disconnect(self.downloadProgress)
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
def sslErrors(self, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v.errorString())
if self.disable_ssl_certificate_validation:
self.reply.ignoreSslErrors()
def abort(self):
"""
Handle request to cancel HTTP call
"""
if (self.reply and self.reply.isRunning()):
self.on_abort = True
self.reply.abort()
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None,
blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
if k and v:
req.setRawHeader(k.encode(), v.encode())
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, io.IOBase):
body = body.read()
if isinstance(body, str):
body = body.encode()
if isinstance(body, dict):
body = str(json.dumps(body)).encode(encoding='utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkReply(self.reply, self.authid)
QgsNetworkAccessManager.instance().setTimeout(int(self.timeout) * 1000)
# necessary to trap local timeout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(
self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
class Nominatim(object):
"""Manage connexion to Nominatim."""
def __init__(self,
url="https://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QgsNetworkAccessManager.instance()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
# query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('q', query)
query_string.addQueryItem('format', 'json')
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, file):
body = body.read()
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)
# necessary to trap local timout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
class NetworkAccessManager(object):
"""
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.
The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.
Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class
Usage 1 (blocking mode)
-----
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException as e:
# Handle exception
pass
Usage 2 (Non blocking mode)
-------------------------
::
NOTE! if blocking mode returns immediatly
it's up to the caller to manage listeners in case
of non blocking mode
nam = NetworkAccessManager(authcgf)
try:
nam.request('http://www.example.com', blocking=False)
nam.reply.finished.connect(a_signal_listener)
except RequestsException as e:
# Handle exception
pass
Get response using method:
nam.httpResult() that return a dictionary with keys:
'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
'content' - bytearray returned from reply
'ok' - request success [True, False]
'headers' - Dicionary containing the reply header
'reason' - fomatted message string with reply.errorString()
'exception' - the exception returne dduring execution
"""
def __init__(self, authid=None, disable_ssl_certificate_validation=False, exception_class=None, debug=False):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.exception_class = exception_class
self.on_abort = False
self.blocking_mode = False
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'content' : '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "NetworkAccessManager")
def httpResult(self):
return self.http_call_result
def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, file):
body = body.read()
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)
# necessary to trap local timout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
#@pyqtSlot()
def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
#self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
pass
#@pyqtSlot(QNetworkReply)
def requestTimedOut(self, QNetworkReply):
"""Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
# adapt http_call_result basing on receiving qgs timer timout signal
self.exception_class = RequestsExceptionTimeout
self.http_call_result.exception = RequestsExceptionTimeout("Timeout error")
#@pyqtSlot(QObject)
def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(k)] = str(v)
self.http_call_result.headers[str(k).lower()] = str(v)
if err != QNetworkReply.NoError:
# handle error
# check if errorString is empty, if so, then set err string as
# reply dump
if re.match('(.)*server replied: $', self.reply.errorString()):
errString = self.reply.errorString() + self.http_call_result.content
else:
errString = self.reply.errorString()
# check if self.http_call_result.status_code is available (client abort
# does not produce http.status_code)
if self.http_call_result.status_code:
msg = "Network error #{0}: {1}".format(
self.http_call_result.status_code, errString)
else:
msg = "Network error: {0}".format(errString)
self.http_call_result.reason = msg
self.http_call_result.ok = False
self.msg_log(msg)
# set return exception
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)
elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(msg)
elif err == QNetworkReply.OperationCanceledError:
# request abort by calling NAM.abort() => cancelled by the user
if self.on_abort:
self.http_call_result.exception = RequestsExceptionUserAbort(msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
self.http_call_result.exception = RequestsException(msg)
# overload exception to the custom exception if available
if self.exception_class:
self.http_call_result.exception = self.exception_class(msg)
else:
# Handle redirections
redirectionUrl = self.reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
if redirectionUrl is not None and redirectionUrl != self.reply.url():
if redirectionUrl.isRelative():
redirectionUrl = self.reply.url().resolved(redirectionUrl)
msg = "Redirected from '{}' to '{}'".format(
self.reply.url().toString(), redirectionUrl.toString())
self.msg_log(msg)
self.reply.deleteLater()
self.reply = None
self.request(redirectionUrl.toString())
# really end request
else:
msg = "Network success #{0}".format(self.reply.error())
self.http_call_result.reason = msg
self.msg_log(msg)
ba = self.reply.readAll()
self.http_call_result.content = bytes(ba)
self.http_call_result.ok = True
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
for k, v in list(self.http_call_result.headers.items()):
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.content) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.content)
else:
self.msg_log("Payload is > 1 KB ...")
# clean reply
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
# Disconnect all slots
self.reply.sslErrors.disconnect(self.sslErrors)
self.reply.finished.disconnect(self.replyFinished)
self.reply.downloadProgress.disconnect(self.downloadProgress)
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
#@pyqtSlot()
def sslErrors(self, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v.errorString())
if self.disable_ssl_certificate_validation:
self.reply.ignoreSslErrors()
#@pyqtSlot()
def abort(self):
"""
Handle request to cancel HTTP call
"""
if (self.reply and self.reply.isRunning()):
self.on_abort = True
self.reply.abort()
def test(self):
# This test is quite fragile as it depends on windows manager behaviour
# regarding focus, so not surprising it doesn't pass
# on other platforms than Linux.
#if 'TRAVIS_OS_NAME' in os.environ and os.environ['TRAVIS_OS_NAME'] == 'osx':
# return
main_dialog = QgsProviderRegistry.instance().selectWidget("WFS")
main_dialog.setProperty("hideDialogs", True)
self.assertIsNotNone(main_dialog)
# Create new connection
btnNew = main_dialog.findChild(QWidget, "btnNew")
self.assertIsNotNone(btnNew)
QTest.mouseClick(btnNew, Qt.LeftButton)
new_conn = find_window('QgsNewHttpConnectionBase')
self.assertIsNotNone(new_conn)
txtName = new_conn.findChild(QLineEdit, "txtName")
self.assertIsNotNone(txtName)
txtName.setText("test_connection")
txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
self.assertIsNotNone(txtUrl)
txtUrl.setText("test_url")
new_conn.accept()
# Wait for object to be destroyed
new_conn = self.wait_object_destruction(new_conn)
# Try to connect
btnConnect = main_dialog.findChild(QWidget, "btnConnect")
self.assertIsNotNone(btnConnect)
QTest.mouseClick(btnConnect, Qt.LeftButton)
# Depends on asynchronous signal
QApplication.processEvents()
error_box = find_window('WFSCapabilitiesErrorBox')
self.assertIsNotNone(error_box)
# Close error box
error_box.accept()
# Wait for object to be destroyed
error_box = self.wait_object_destruction(error_box)
# Edit connection
btnEdit = main_dialog.findChild(QWidget, "btnEdit")
self.assertIsNotNone(btnEdit)
QTest.mouseClick(btnEdit, Qt.LeftButton)
new_conn = find_window('QgsNewHttpConnectionBase',)
self.assertIsNotNone(new_conn)
txtName = new_conn.findChild(QLineEdit, "txtName")
self.assertIsNotNone(txtName)
txtName.setText("test_connection")
txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
self.assertIsNotNone(txtUrl)
endpoint = self.basetestpath + '/fake_qgis_http_endpoint'
expected_endpoint = endpoint
if sys.platform == 'win32' and expected_endpoint[1] == ':':
expected_endpoint = expected_endpoint[0] + expected_endpoint[2:]
with open(sanitize(endpoint, '?SERVICE=WFS?REQUEST=GetCapabilities?ACCEPTVERSIONS=2.0.0,1.1.0,1.0.0'), 'wb') as f:
f.write("""
<wfs:WFS_Capabilities version="2.0.0" xmlns="http://www.opengis.net/wfs/2.0" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://schemas.opengis.net/gml/3.2" xmlns:fes="http://www.opengis.net/fes/2.0">
<FeatureTypeList>
<FeatureType>
<Name>my:typename</Name>
<Title>Title</Title>
<Abstract>Abstract</Abstract>
<DefaultCRS>urn:ogc:def:crs:EPSG::4326</DefaultCRS>
<ows:WGS84BoundingBox>
<ows:LowerCorner>-71.123 66.33</ows:LowerCorner>
<ows:UpperCorner>-65.32 78.3</ows:UpperCorner>
</ows:WGS84BoundingBox>
</FeatureType>
</FeatureTypeList>
<fes:Filter_Capabilities>
<fes:Spatial_Capabilities>
<fes:GeometryOperands>
<fes:GeometryOperand name="gml:Envelope"/>
<fes:GeometryOperand name="gml:Point"/>
<fes:GeometryOperand name="gml:MultiPoint"/>
<fes:GeometryOperand name="gml:LineString"/>
<fes:GeometryOperand name="gml:MultiLineString"/>
<fes:GeometryOperand name="gml:Polygon"/>
<fes:GeometryOperand name="gml:MultiPolygon"/>
<fes:GeometryOperand name="gml:MultiGeometry"/>
</fes:GeometryOperands>
<fes:SpatialOperators>
<fes:SpatialOperator name="Disjoint"/>
<fes:SpatialOperator name="Equals"/>
<fes:SpatialOperator name="DWithin"/>
<fes:SpatialOperator name="Beyond"/>
<fes:SpatialOperator name="Intersects"/>
<fes:SpatialOperator name="Touches"/>
<fes:SpatialOperator name="Crosses"/>
<fes:SpatialOperator name="Within"/>
<fes:SpatialOperator name="Contains"/>
<fes:SpatialOperator name="Overlaps"/>
<fes:SpatialOperator name="BBOX"/>
</fes:SpatialOperators>
</fes:Spatial_Capabilities>
<fes:Functions>
<fes:Function name="abs">
<fes:Returns>xs:int</fes:Returns>
<fes:Arguments>
<fes:Argument name="param">
<fes:Type>xs:int</fes:Type>
</fes:Argument>
</fes:Arguments>
</fes:Function>
</fes:Functions>
</fes:Filter_Capabilities>
</wfs:WFS_Capabilities>""".encode('UTF-8'))
txtUrl.setText("http://" + endpoint)
new_conn.accept()
# Wait for object to be destroyed
new_conn = self.wait_object_destruction(new_conn)
# Try to connect
btnConnect = main_dialog.findChild(QWidget, "btnConnect")
self.assertIsNotNone(btnConnect)
QTest.mouseClick(btnConnect, Qt.LeftButton)
# We need to show (and raise for Mac) the dialog so that the focus changes
self.loop = QEventLoop()
treeView = main_dialog.findChild(QTreeView, "treeView")
treeView.selectionModel().currentRowChanged.connect(main_dialog.hide)
treeView.selectionModel().currentRowChanged.connect(self.loop.quit)
main_dialog.show()
main_dialog.raise_()
self.loop.exec_()
# Add layer
buttonAdd = self.get_button_add(main_dialog)
self.assertTrue(buttonAdd.isEnabled())
self.addWfsLayer_uri = None
self.addWfsLayer_layer_name = None
main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
QTest.mouseClick(buttonAdd, Qt.LeftButton)
self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=')
self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
# Click on Build Query
buttonBuildQuery = self.get_button_build_query(main_dialog)
self.assertTrue(buttonBuildQuery.isEnabled())
QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
error_box = find_window('WFSFeatureTypeErrorBox')
self.assertIsNotNone(error_box)
# Close error box
error_box.accept()
# Wait for object to be destroyed
error_box = self.wait_object_destruction(error_box)
# Click again but with valid DescribeFeatureType
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAME=my:typename'), 'wb') as f:
f.write("""
<xsd:schema xmlns:my="http://my" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://my">
<xsd:import namespace="http://www.opengis.net/gml/3.2"/>
<xsd:complexType name="typenameType">
<xsd:complexContent>
<xsd:extension base="gml:AbstractFeatureType">
<xsd:sequence>
<xsd:element maxOccurs="1" minOccurs="0" name="intfield" nillable="true" type="xsd:int"/>
<xsd:element maxOccurs="1" minOccurs="0" name="geometryProperty" nillable="true" type="gml:PolygonPropertyType"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:element name="typename" substitutionGroup="gml:_Feature" type="my:typenameType"/>
</xsd:schema>
""".encode('UTF-8'))
QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
# Check that the combos are properly initialized
dialog = find_window('QgsSQLComposerDialogBase')
self.assertIsNotNone(dialog)
mTablesCombo = dialog.findChild(QComboBox, "mTablesCombo")
self.assertIsNotNone(mTablesCombo)
self.assertEqual(mTablesCombo.itemText(1), 'typename (Title)')
mColumnsCombo = dialog.findChild(QComboBox, "mColumnsCombo")
self.assertIsNotNone(mColumnsCombo)
self.assertEqual(mColumnsCombo.itemText(1), 'intfield (int)')
self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 2), 'geometryProperty (geometry)')
self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 1), '*')
mFunctionsCombo = dialog.findChild(QComboBox, "mFunctionsCombo")
self.assertIsNotNone(mFunctionsCombo)
self.assertEqual(mFunctionsCombo.itemText(1), 'abs(param: int): int')
mSpatialPredicatesCombo = dialog.findChild(QComboBox, "mSpatialPredicatesCombo")
self.assertIsNotNone(mSpatialPredicatesCombo)
self.assertEqual(mSpatialPredicatesCombo.itemText(1), 'ST_Disjoint(geometry, geometry): boolean')
mWhereEditor = dialog.findChild(QTextEdit, "mWhereEditor")
self.assertIsNotNone(mWhereEditor)
mWhereEditor.setText('1 = 1')
dialog.accept()
# Wait for object to be destroyed
dialog = self.wait_object_destruction(dialog)
# Add layer
buttonAdd = self.get_button_add(main_dialog)
self.assertTrue(buttonAdd.isEnabled())
self.addWfsLayer_uri = None
self.addWfsLayer_layer_name = None
main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
QTest.mouseClick(buttonAdd, Qt.LeftButton)
self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=SELECT * FROM typename WHERE 1 = 1')
self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
class NetworkAccessManager():
"""
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.
The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.
Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class
Usage
-----
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException, e:
# Handle exception
pass
"""
def __init__(self,
authid=None,
disable_ssl_certificate_validation=False,
exception_class=None,
debug=True):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.exception_class = exception_class
self.cookie = None
self.basicauth = None
def setBasicauth(self, encodedString):
self.basicauth = encodedString
def setCookie(self, cookie):
self.cookie = cookie
def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "NetworkAccessManager")
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'text': '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
req = QNetworkRequest()
# Avoid double quoting form QUrl
if PYTHON_VERSION >= 3:
url = urllib.parse.unquote(url)
else:
url = urllib2.unquote(url)
req.setUrl(QUrl(url))
if self.cookie is not None:
if headers is not None:
headers['Cookie'] = self.cookie
else:
headers = {'Cookie': self.cookie}
if self.basicauth is not None:
if headers is not None:
headers['Authorization'] = self.basicauth
else:
headers = {'Authorization': self.basicauth}
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in headers.items():
if PYTHON_VERSION >= 3:
if isinstance(k, str):
k = k.encode('utf-8')
if isinstance(v, str):
v = v.encode('utf-8')
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if PYTHON_VERSION >= 3:
if isinstance(body, str):
body = body.encode('utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(
self.reply, self.authid)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
self.reply.downloadProgress.connect(self.downloadProgress)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_()
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
headers = {
str(h): str(self.reply.rawHeader(h))
for h in self.reply.rawHeaderList()
}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.text) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")
except Exception as e:
raise e
finally:
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.text)
#@pyqtSlot()
def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
#self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
pass
#@pyqtSlot()
def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(
QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(k)] = str(v)
self.http_call_result.headers[str(k).lower()] = str(v)
if err != QNetworkReply.NoError:
msg = "Network error #{0}: {1}".format(self.reply.error(),
self.reply.errorString())
self.http_call_result.reason = msg
self.http_call_result.ok = False
self.msg_log(msg)
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)
elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(
msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
# since Python 3 readAll() returns a PyQt5.QByteArray, we
# want only the data
if PYTHON_VERSION >= 3:
self.http_call_result.text = self.reply.readAll().data(
).decode('utf-8')
else:
self.http_call_result.text = str(self.reply.readAll())
self.http_call_result.ok = True
self.reply.deleteLater()
#@pyqtSlot()
def sslErrors(self, reply, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v)
if self.disable_ssl_certificate_validation:
reply.ignoreSslErrors()
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'text': '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
req = QNetworkRequest()
# Avoid double quoting form QUrl
if PYTHON_VERSION >= 3:
url = urllib.parse.unquote(url)
else:
url = urllib2.unquote(url)
req.setUrl(QUrl(url))
if self.cookie is not None:
if headers is not None:
headers['Cookie'] = self.cookie
else:
headers = {'Cookie': self.cookie}
if self.basicauth is not None:
if headers is not None:
headers['Authorization'] = self.basicauth
else:
headers = {'Authorization': self.basicauth}
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in headers.items():
if PYTHON_VERSION >= 3:
if isinstance(k, str):
k = k.encode('utf-8')
if isinstance(v, str):
v = v.encode('utf-8')
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if PYTHON_VERSION >= 3:
if isinstance(body, str):
body = body.encode('utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(
self.reply, self.authid)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
self.reply.downloadProgress.connect(self.downloadProgress)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_()
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
headers = {
str(h): str(self.reply.rawHeader(h))
for h in self.reply.rawHeaderList()
}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.text) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")
except Exception as e:
raise e
finally:
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.text)
def testStoreFetchFileImmediately(self):
"""
Test file storing and fetching (Immediately mode)
"""
f = self.getNewFile(b"New content")
# store
url = self.url + "/" + os.path.basename(f.name)
storedContent = self.storage.store(f.name, url, self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(storedContent)
self.assertEqual(storedContent.status(), Qgis.ContentStatus.Running)
spyErrorOccurred = QSignalSpy(storedContent.errorOccurred)
spyProgressChanged = QSignalSpy(storedContent.progressChanged)
loop = QEventLoop()
storedContent.stored.connect(loop.quit)
storedContent.errorOccurred.connect(loop.quit)
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(storedContent.url(), url)
self.assertFalse(storedContent.errorString())
self.assertEqual(storedContent.status(), Qgis.ContentStatus.Finished)
self.assertTrue(len(spyProgressChanged) > 0)
self.assertEqual(spyProgressChanged[-1][0], 100)
# fetch
fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(fetchedContent)
# Some external storage (SimpleCopy) doesn't actually need to retrieve the resource
self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Finished or
fetchedContent.status() == Qgis.ContentStatus.Running)
if (fetchedContent.status() == Qgis.ContentStatus.Running):
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.fetched.connect(loop.quit)
fetchedContent.errorOccurred.connect(loop.quit)
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
self.assertFalse(fetchedContent.errorString())
self.assertTrue(fetchedContent.filePath())
self.checkContent(fetchedContent.filePath(), b"New content")
self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')
# fetch again, should be cached
fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(fetchedContent)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
self.assertTrue(not fetchedContent.errorString())
self.assertTrue(fetchedContent.filePath())
self.checkContent(fetchedContent.filePath(), b"New content")
self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')
# fetch bad url
fetchedContent = self.storage.fetch(self.url + "/error", self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(fetchedContent)
# Some external storage (SimpleCopy) doesn't actually need to retrieve the resource
self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Failed or
fetchedContent.status() == Qgis.ContentStatus.Running)
if (fetchedContent.status() == Qgis.ContentStatus.Running):
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.errorOccurred.connect(loop.quit)
fetchedContent.fetched.connect(loop.quit)
loop.exec()
self.assertEqual(len(spyErrorOccurred), 1)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Failed)
self.assertTrue(fetchedContent.errorString())
self.assertFalse(fetchedContent.filePath())
def getVideoLocationInfo(videoPath, dataFile=False):
""" Get basic location info about the video """
location = []
try:
if dataFile == False:
p = _spawn([
'-i', videoPath, '-ss', '00:00:00', '-to', '00:00:01', '-map',
'data-re', '-f', 'data', '-'
])
stdout_data, _ = p.communicate()
################
else:
global listOfMetadata
stdout_data = listOfMetadata[0]
################
if stdout_data == b'':
return
for packet in StreamParser(stdout_data):
if isinstance(packet, UnknownElement):
qgsu.showUserAndLogMessage(
"Error interpreting klv data, metadata cannot be read.",
"the parser did not recognize KLV data",
level=QGis.Warning)
continue
packet.MetadataList()
frameCenterLat = packet.FrameCenterLatitude
frameCenterLon = packet.FrameCenterLongitude
loc = "-"
if Reverse_geocoding_url != "":
try:
url = QUrl(
Reverse_geocoding_url.format(str(frameCenterLat),
str(frameCenterLon)))
request = QNetworkRequest(url)
reply = QgsNetworkAccessManager.instance().get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
reply.finished.disconnect(loop.quit)
loop = None
result = reply.readAll()
data = json.loads(result.data())
if "village" in data["address"] and "state" in data[
"address"]:
loc = data["address"]["village"] + \
", " + data["address"]["state"]
elif "town" in data["address"] and "state" in data[
"address"]:
loc = data["address"]["town"] + \
", " + data["address"]["state"]
else:
loc = data["display_name"]
except Exception:
qgsu.showUserAndLogMessage(
"",
"getVideoLocationInfo: failed to get address from reverse geocoding service.",
onlyLog=True)
location = [frameCenterLat, frameCenterLon, loc]
qgsu.showUserAndLogMessage(
"",
"Got Location: lon: " + str(frameCenterLon) + " lat: " +
str(frameCenterLat) + " location: " + str(loc),
onlyLog=True)
break
else:
qgsu.showUserAndLogMessage(
QCoreApplication.translate(
"QgsFmvUtils", "This video doesn't have Metadata ! : "))
except Exception as e:
qgsu.showUserAndLogMessage(
QCoreApplication.translate("QgsFmvUtils",
"Video info callback failed! : "),
str(e))
return location
class Downloader(QObject):
NOT_FOUND = 0
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
replyFinished = pyqtSignal(str, int, int)
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.queue = []
self.redirected_urls = {}
self.requestingUrls = []
self.replies = []
self.eventLoop = QEventLoop()
self.sync = False
self.fetchedFiles = {}
self.clearCounts()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.fetchTimedOut)
# network settings
self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
self.max_connection = 4
self.default_cache_expiration = 24
self.errorStatus = Downloader.NO_ERROR
def clearCounts(self):
self.fetchSuccesses = 0
self.fetchErrors = 0
self.cacheHits = 0
def fetchTimedOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
def abort(self):
# clear queue and abort sent requests
self.queue = []
self.timer.stop()
for reply in self.replies:
reply.abort()
self.errorStatus = Downloader.UNKNOWN_ERROR
def replyFinishedSlot(self):
reply = self.sender()
url = reply.request().url().toString()
self.log("replyFinishedSlot: %s" % url)
if not url in self.fetchedFiles:
self.fetchedFiles[url] = None
self.requestingUrls.remove(url)
self.replies.remove(reply)
isFromCache = 0
httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if reply.error() == QNetworkReply.NoError:
if httpStatusCode == 301:
new_url = str(reply.rawHeader("Location"))
self.addToQueue(new_url, url)
else:
self.fetchSuccesses += 1
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self.cacheHits += 1
isFromCache = 1
elif not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
# self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
if metadata.expirationDate().isNull():
metadata.setExpirationDate(
QDateTime.currentDateTime().addSecs(self.default_cache_expiration * 60 * 60))
cache.updateMetaData(metadata)
self.log(
"Default expiration date has been set: %s (%d h)" % (url, self.default_cache_expiration))
if reply.isReadable():
data = reply.readAll()
if self.redirected_urls.has_key(url):
url = self.redirected_urls[url]
self.fetchedFiles[url] = data
else:
qDebug("http status code: " + str(httpStatusCode))
# self.emit(SIGNAL('replyFinished(QString, int, int)'), url, reply.error(), isFromCache)
self.replyFinished.emit(url, reply.error(), isFromCache)
else:
if self.sync and httpStatusCode == 404:
self.fetchedFiles[url] = self.NOT_FOUND
self.fetchErrors += 1
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
reply.deleteLater()
if debug_mode:
qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls)))
if len(self.queue) + len(self.requestingUrls) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
self.log("replyFinishedSlot End: %s" % url)
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
request = QNetworkRequest(QUrl(url))
request.setRawHeader("User-Agent", self.userAgent)
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self.replyFinishedSlot)
self.requestingUrls.append(url)
self.replies.append(reply)
return reply
def fetchFiles(self, urlList, timeout_ms=0):
self.log("fetchFiles()")
self.sync = True
self.queue = []
self.redirected_urls = {}
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
self.log("fetchFiles() End: %d" % self.errorStatus)
if timeout_ms > 0:
self.timer.stop()
return self.fetchedFiles
def addToQueue(self, url, redirected_from=None):
if url in self.queue:
return False
self.queue.append(url)
if redirected_from is not None:
self.redirected_urls[url] = redirected_from
return True
def queueCount(self):
return len(self.queue)
def finishedCount(self):
return len(self.fetchedFiles)
def unfinishedCount(self):
return len(self.queue) + len(self.requestingUrls)
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def fetchFilesAsync(self, urlList, timeout_ms=0):
self.log("fetchFilesAsync()")
self.sync = False
self.queue = []
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()