def wait_object_destruction(self, my_object):
loop = QEventLoop()
name = my_object.objectName()
my_object.destroyed.connect(loop.quit)
loop.exec_()
self.assertIsNone(find_window(name))
return None
def testInvalidAuthFileDownload(self):
"""
Download a protected map tile without authcfg
"""
qs = "?" + "&".join(["%s=%s" % i for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, None, False)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
self.assertTrue(self.error_was_called)
self.assertTrue("Download failed: Host requires authentication" in str(self.error_args), "Error args is: %s" % str(self.error_args))
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(
self._url, self.result_path, delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
else:
return self.result_path
def __sync_request(self, url):
_url = QUrl(url)
_request = QNetworkRequest(_url)
self.__replies.append(_request)
QgsNetworkAccessManager.instance().sslErrors.connect(self.__supress_ssl_errors)
_reply = QgsNetworkAccessManager.instance().get(_request)
# wait
loop = QEventLoop()
_reply.finished.connect(loop.quit)
loop.exec_()
_reply.finished.disconnect(loop.quit)
QgsNetworkAccessManager.instance().sslErrors.disconnect(self.__supress_ssl_errors)
loop = None
error = _reply.error()
if error != QNetworkReply.NoError:
raise Exception(error)
result_code = _reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
result = _reply.readAll()
self.__replies.append(_reply)
_reply.deleteLater()
if result_code in [301, 302, 307]:
redirect_url = _reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
return self.__sync_request(redirect_url)
else:
return result
def testValidAuthFileDownload(self):
"""
Download a map tile with valid authcfg
"""
qs = "?" + "&".join([
"%s=%s" % i
for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())
])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination,
self.auth_config.id(), False)
downloader.downloadCompleted.connect(
partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot,
'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot,
'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
# Check the we've got a likely PNG image
self.assertTrue(self.completed_was_called)
self.assertTrue(
os.path.getsize(destination) > 2000,
"Image size: %s" % os.path.getsize(destination)) # > 1MB
with open(destination, 'rb') as f:
self.assertTrue(b'PNG' in f.read()) # is a PNG
def fetchFiles(self, urls):
self.logT("TileLayer.fetchFiles() starts")
# create a QEventLoop object that belongs to the current thread (if ver. > 2.1, it is render thread)
eventLoop = QEventLoop()
self.logT("Create event loop: " + str(eventLoop)) # DEBUG
# QObject.connect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
self.allRepliesFinished.connect(eventLoop.quit)
# create a timer to watch whether rendering is stopped
watchTimer = QTimer()
watchTimer.timeout.connect(eventLoop.quit)
# send a fetch request to the main thread
# self.emit(SIGNAL("fetchRequest(QStringList)"), urls)
self.fetchRequest.emit(urls)
# wait for the fetch to finish
tick = 0
interval = 500
timeoutTick = self.downloadTimeout / interval
watchTimer.start(interval)
while tick < timeoutTick:
# run event loop for 0.5 seconds at maximum
eventLoop.exec_()
if debug_mode:
qDebug("watchTimerTick: %d" % tick)
qDebug("unfinished downloads: %d" % self.downloader.unfinishedCount())
if self.downloader.unfinishedCount() == 0 or self.renderContext.renderingStopped():
break
tick += 1
watchTimer.stop()
if tick == timeoutTick and self.downloader.unfinishedCount() > 0:
self.log("fetchFiles timeout")
# self.emitShowBarMessage("fetchFiles timeout", duration=5) #DEBUG
self.downloader.abort()
self.downloader.errorStatus = Downloader.TIMEOUT_ERROR
files = self.downloader.fetchedFiles
watchTimer.timeout.disconnect(eventLoop.quit) #
# QObject.disconnect(self, SIGNAL("allRepliesFinished()"), eventLoop.quit)
self.allRepliesFinished.disconnect(eventLoop.quit)
self.logT("TileLayer.fetchFiles() ends")
return files
def __init__(self, worker_class, process_name, *args):
self.exception = None
self.success = None
self.return_val = None
self.worker = worker_class(*args)
pause = QEventLoop()
self.worker.finished.connect(pause.quit)
self.worker.successfully_finished.connect(self.save_success)
self.worker.error.connect(self.save_exception)
start_worker(self.worker, iface,
QApplication.translate("MISLAND", u'Processing: {}'.format(process_name)))
pause.exec_()
if self.exception:
raise self.exception
def _download_qgis(packageUrl, handle):
from qgis.core import QgsNetworkAccessManager
request = QNetworkRequest(QUrl(packageUrl))
reply = QgsNetworkAccessManager.instance().get(request)
evloop = QEventLoop()
reply.finished.connect(evloop.quit)
evloop.exec_(QEventLoop.ExcludeUserInputEvents)
content_type = reply.rawHeader('Content-Type')
if bytearray(content_type) == bytearray('text/plain; charset=utf-8'):
handle.write(bytearray(reply.readAll()))
else:
msg = 'Failed to download %s\n\nPlease check your QGIS network settings and authentication db' % (
packageUrl)
ret_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if ret_code:
msg += '\n\nThe HTTP status code was %d.' % (ret_code)
raise Exception(msg)
def get_available_maps(maps_uri):
"""Fetch the list of available and QGIS supported maps from BCS endpoint,
apparently this API method does not require auth"""
# For testing purposes, we can also access to a json file directly
if not maps_uri.startswith('http'):
j = json.load(open(maps_uri))
else:
t = mktemp()
q = QgsFileDownloader(QUrl(maps_uri), t)
loop = QEventLoop()
q.downloadExited.connect(loop.quit)
loop.exec_()
if not os.path.isfile(t):
return []
with open(t) as f:
j = json.load(f)
os.unlink(t)
return [l for l in j if layer_is_supported(l)]
def testInvalidAuthFileDownload(self):
"""
Download a protected map tile without authcfg
"""
qs = "?" + "&".join([
"%s=%s" % i
for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())
])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, None, False)
downloader.downloadCompleted.connect(
partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot,
'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot,
'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
self.assertTrue(self.error_was_called)
self.assertTrue(
"Download failed: Host requires authentication"
in str(self.error_args),
"Error args is: %s" % str(self.error_args))
def download_image(self, url):
res = False
img = None
msg = {'text': '', 'level': Qgis.Warning}
if url:
self.logger.info(__name__, "Downloading file from {}".format(url))
msg_status_bar = "Downloading image from document repository (this might take a while)..."
with ProcessWithStatus(msg_status_bar):
if is_connected(TEST_SERVER):
nam = QNetworkAccessManager()
request = QNetworkRequest(QUrl(url))
reply = nam.get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
status = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if status == 200:
res = True
img = reply.readAll()
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to the server. The server might be down or the service cannot be reached at the given URL."
)
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to Internet.")
else:
res = False
msg['text'] = QCoreApplication.translate("SettingsDialog",
"Not valid URL")
if not res:
self.logger.log_message(__name__, msg['text'], msg['level'])
return res, img
def process_put_call(self, url, data=None, report_url=True):
"""
Run a PUT request and return reply data
:param url: url for request
:param data:
:param report_url: True if URL should be reported to feedback
:return: response or error message in json format
"""
if self.connection.read_only:
return {
"error": {
"msg": "Graphium connection is set to read-only!"
}
}
url_query = QUrl(url)
if report_url:
self.report_info('PUT ' + url_query.toString())
data_byte_array = QJsonDocument.fromVariant(data)
request = QNetworkRequest(url_query)
if self.connection.auth_cfg != '':
self.auth = 0
config = QgsAuthMethodConfig()
QgsApplication.authManager().loadAuthenticationConfig(
self.connection.auth_cfg, config, True)
concatenated = config.configMap(
)['username'] + ":" + config.configMap()['password']
data = base64.encodebytes(concatenated.encode("utf-8")).replace(
'\n'.encode("utf-8"), ''.encode("utf-8"))
request.setRawHeader("Authorization".encode("utf-8"),
("Basic %s" % data).encode("utf-8"))
request.setRawHeader("Accept".encode("utf-8"),
"*/*".encode("utf-8"))
loop = QEventLoop() # https://stackoverflow.com/a/46514984
reply = self.network_access_manager.put(request,
data_byte_array.toJson())
reply.finished.connect(loop.quit)
loop.exec_()
return self.process_q_reply(reply)
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language", bytes(settings.value("default_language", "fr"), "utf8"))
req.setRawHeader(b"User-Agent", bytes(settings.value('http_user_agent', plugin_name()), "utf8"))
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def test_api_usage_async(self):
"""Test standard async API usage"""
api = NzElectoralApi('http://localhost:%s' % self.port)
nam = api.status()
self.last_result = ''
expected = {
"version": self.API_VERSION,
"gmsVersion": "LINZ_Output_20180108_2018_V1_00"
}
el = QEventLoop()
def f(nam):
"""Wrapper"""
self.last_result = api.parse_async(nam)['content']
nam.reply.finished.connect(partial(f, nam))
nam.reply.finished.connect(el.quit)
el.exec_(QEventLoop.ExcludeUserInputEvents)
self.assertEqual(self.last_result, expected)
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language", b"fr")
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
raise OverpassTimeoutException
# The download is done, checking for not complete OSM file.
# Overpass might aborted the request with HTTP 200.
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = (
'<remark> runtime error: Query timed out in "[a-z]+" at line '
'[\d]+ after ([\d]+) seconds. </remark>')
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
# Everything went fine
return self.result_path
def get_available_providers(providers_uri):
"""Fetch the list of available providers from BCS endpoint,
apparently this API method does not require auth"""
# For testing purposes, we can also access to a json file directly
if not providers_uri.startswith('http'):
try:
j = json.load(open(providers_uri, encoding='utf-8'))
except:
j = json.load(open(providers_uri))
else:
t = mktemp()
q = QgsFileDownloader(QUrl(providers_uri), t)
loop = QEventLoop()
q.downloadExited.connect(loop.quit)
loop.exec_()
if not os.path.isfile(t):
return []
with open(t) as f:
j = json.load(f)
os.unlink(t)
return j
class OpenlayersRenderer(QgsMapLayerRenderer):
def __init__(self, layer, context, webPage, layerType):
""" Initialize the object. This function is still run in the GUI thread.
Should refrain from doing any heavy work.
"""
QgsMapLayerRenderer.__init__(self, layer.id())
self.context = context
self.controller = OpenlayersController(None, context, webPage,
layerType)
self.loop = None
def render(self):
""" do the rendering. This function is called in the worker thread """
debug("[WORKER THREAD] Calling request() asynchronously", 3)
QMetaObject.invokeMethod(self.controller, "request")
# setup a timer that checks whether the rendering has not been stopped
# in the meanwhile
timer = QTimer()
timer.setInterval(50)
timer.timeout.connect(self.onTimeout)
timer.start()
debug("[WORKER THREAD] Waiting for the async request to complete", 3)
self.loop = QEventLoop()
self.controller.finished.connect(self.loop.exit)
self.loop.exec_()
debug("[WORKER THREAD] Async request finished", 3)
painter = self.context.painter()
painter.drawImage(0, 0, self.controller.img)
return True
def onTimeout(self):
""" periodically check whether the rendering should not be stopped """
if self.context.renderingStopped():
debug("[WORKER THREAD] Cancelling rendering", 3)
self.loop.exit()
def process_delete_call(self, url, report_url=True):
"""
Run a DELETE request and return reply data
:param url: url for request
:param report_url: True if URL should be reported to feedback
:return: response or error message in json format
"""
if self.connection.read_only:
return {
"error": {
"msg": "Graphium connection is set to read-only!"
}
}
url_query = QUrl(url)
if report_url:
self.report_info('DELETE ' + url_query.toString())
request = QNetworkRequest(url_query)
if self.connection.auth_cfg != '':
self.auth = 0
config = QgsAuthMethodConfig()
QgsApplication.authManager().loadAuthenticationConfig(
self.connection.auth_cfg, config, True)
concatenated = config.configMap(
)['username'] + ":" + config.configMap()['password']
data = base64.encodebytes(concatenated.encode("utf-8")).replace(
'\n'.encode("utf-8"), ''.encode("utf-8"))
request.setRawHeader("Authorization".encode("utf-8"),
("Basic %s" % data).encode("utf-8"))
request.setRawHeader("Accept".encode("utf-8"),
"*/*".encode("utf-8"))
loop = QEventLoop()
reply = self.network_access_manager.deleteResource(request)
reply.finished.connect(loop.quit)
loop.exec_()
return self.process_q_reply(reply)
class OpenlayersRenderer(QgsMapLayerRenderer):
def __init__(self, layer, context, webPage, layerType):
""" Initialize the object. This function is still run in the GUI thread.
Should refrain from doing any heavy work.
"""
QgsMapLayerRenderer.__init__(self, layer.id())
self.context = context
self.controller = OpenlayersController(None,
context, webPage, layerType)
self.loop = None
def render(self):
""" do the rendering. This function is called in the worker thread """
debug("[WORKER THREAD] Calling request() asynchronously", 3)
QMetaObject.invokeMethod(self.controller, "request")
# setup a timer that checks whether the rendering has not been stopped
# in the meanwhile
timer = QTimer()
timer.setInterval(50)
timer.timeout.connect(self.onTimeout)
timer.start()
debug("[WORKER THREAD] Waiting for the async request to complete", 3)
self.loop = QEventLoop()
self.controller.finished.connect(self.loop.exit)
self.loop.exec_()
debug("[WORKER THREAD] Async request finished", 3)
painter = self.context.painter()
painter.drawImage(0, 0, self.controller.img)
return True
def onTimeout(self):
""" periodically check whether the rendering should not be stopped """
if self.context.renderingStopped():
debug("[WORKER THREAD] Cancelling rendering", 3)
self.loop.exit()
def _make_download(self, url, destination, cancel=False):
self.completed_was_called = False
self.error_was_called = False
self.canceled_was_called = False
self.progress_was_called = False
self.exited_was_called = False
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
if cancel:
downloader.downloadProgress.connect(downloader.cancelDownload)
loop.exec_()
def _make_download(self, url, destination, cancel=False):
self.completed_was_called = False
self.error_was_called = False
self.canceled_was_called = False
self.progress_was_called = False
self.exited_was_called = False
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
if cancel:
downloader.downloadProgress.connect(downloader.cancelDownload)
loop.exec_()
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
for message in self.errors:
self.is_query_timed_out(message)
self.is_bad_request(message)
LOGGER.error(message)
if len(self.errors):
raise NetWorkErrorException('Overpass API', ', '.join(self.errors))
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
# Do not raise a QuickOSM exception here
# It must be a bug from QuickOSM
raise FileNotFoundError
self.check_file(self.result_path)
# Everything went fine
return self.result_path
def testValidAuthFileDownload(self):
"""
Download a map tile with valid authcfg
"""
qs = "?" + "&".join(["%s=%s" % i for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, self.auth_config.id(), False)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
# Check the we've got a likely PNG image
self.assertTrue(self.completed_was_called)
self.assertTrue(os.path.getsize(destination) > 700000, "Image size: %s" % os.path.getsize(destination)) # > 1MB
with open(destination, 'rb') as f:
self.assertTrue(b'PNG' in f.read()) # is a PNG
def loadModel(self):
imd = None
try:
loader = ModelLoader(self.ui.mDataLineEdit.text())
models = loader.detect_models()
model_names = map(lambda m: m.name, models)
self._log_output("Looking up models: " + ', '.join(model_names))
ili = loader.gen_lookup_ili()
qDebug(ili)
wpsreq = self._create_wps_request(ili)
url = self.ui.mIlisMetaUrlLineEdit.text()
req = QNetworkRequest(QUrl(url))
req.setHeader(QNetworkRequest.ContentTypeHeader, 'application/xml')
reply = QgsNetworkAccessManager.instance().post(req, wpsreq)
# Wait for reply or timeout
loop = QEventLoop()
reply.finished.connect(loop.quit)
QTimer.singleShot(15000, reply.abort)
loop.exec_()
if reply.isFinished() and reply.error() == QNetworkReply.NoError:
result = reply.readAll()
imd = self._parse_wps_response(result)
except:
qDebug("Exception during IlisModel download")
if imd is None:
self._show_log_window()
QgsApplication.messageLog().logMessage(
"Couldn't download Ilismeta model", "Interlis",
Qgis.MessageLevel(1))
self.ui.mModelLineEdit.setText("")
else:
fh, imdfn = tempfile.mkstemp(suffix='.imd')
os.close(fh)
with codecs.open(imdfn, "w", encoding='utf-8') as file:
file.write(imd)
self.ui.mModelLineEdit.setText(imdfn)
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language",
bytes(settings.value("default_language", "fr"), "utf8"))
req.setRawHeader(
b"User-Agent",
bytes(settings.value("http_user_agent", plugin_name()), "utf8"))
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def testPythonCreateChildrenCalledFromCplusplus(self):
""" test createChildren() method implemented in Python, called from C++ """
loop = QEventLoop()
NUM_ITERS = 10 # set more to detect memory leaks
for i in range(NUM_ITERS):
tabSetDestroyedFlag = [False]
item = PyQgsDataConnectionItem(None, "name", "", "my_provider")
item.tabSetDestroyedFlag = tabSetDestroyedFlag
# Causes PyQgsDataConnectionItem.createChildren() to be called
item.populate()
# wait for populate() to have done its job
item.stateChanged.connect(loop.quit)
loop.exec_()
# Python object PyQgsLayerItem should still be alive
self.assertFalse(tabSetDestroyedFlag[0])
children = item.children()
self.assertEqual(len(children), 2)
self.assertEqual(children[0].name(), "name")
self.assertEqual(children[1].name(), "name2")
del (children)
# Delete the object and make sure all deferred deletions are processed
item.destroyed.connect(loop.quit)
item.deleteLater()
loop.exec_()
# Check that the PyQgsLayerItem Python object is now destroyed
self.assertTrue(tabSetDestroyedFlag[0])
tabSetDestroyedFlag[0] = False
def is_source_service_valid(self):
res = False
msg = {'text': '', 'level': Qgis.Warning}
url = self.txt_service_endpoint.text().strip()
if url:
with OverrideCursor(Qt.WaitCursor):
self.qgis_utils.status_bar_message_emitted.emit(
"Checking source service availability (this might take a while)...",
0)
QCoreApplication.processEvents()
if self.qgis_utils.is_connected(TEST_SERVER):
nam = QNetworkAccessManager()
request = QNetworkRequest(QUrl(url))
reply = nam.get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
allData = reply.readAll()
response = QTextStream(allData, QIODevice.ReadOnly)
status = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if status == 200:
try:
data = json.loads(response.readAll())
if 'id' in data and data[
'id'] == SOURCE_SERVICE_EXPECTED_ID:
res = True
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"The tested service is valid to upload files!"
)
msg['level'] = Qgis.Info
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"The tested upload service is not compatible: no valid 'id' found in response."
)
except json.decoder.JSONDecodeError as e:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"Response from the tested service is not compatible: not valid JSON found."
)
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to the server. The server might be down or the service cannot be reached at the given URL."
)
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to Internet.")
self.qgis_utils.clear_status_bar_emitted.emit()
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog", "Not valid service URL to test!")
return (res, msg)
class NetworkAccessManager():
"""
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.
The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.
Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class
Usage
-----
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException, e:
# Handle exception
pass
"""
def __init__(self,
authid=None,
disable_ssl_certificate_validation=False,
exception_class=None,
debug=True):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.exception_class = exception_class
self.cookie = None
self.basicauth = None
def setBasicauth(self, encodedString):
self.basicauth = encodedString
def setCookie(self, cookie):
self.cookie = cookie
def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "NetworkAccessManager")
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'text': '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
req = QNetworkRequest()
# Avoid double quoting form QUrl
if PYTHON_VERSION >= 3:
url = urllib.parse.unquote(url)
else:
url = urllib2.unquote(url)
req.setUrl(QUrl(url))
if self.cookie is not None:
if headers is not None:
headers['Cookie'] = self.cookie
else:
headers = {'Cookie': self.cookie}
if self.basicauth is not None:
if headers is not None:
headers['Authorization'] = self.basicauth
else:
headers = {'Authorization': self.basicauth}
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in headers.items():
if PYTHON_VERSION >= 3:
if isinstance(k, str):
k = k.encode('utf-8')
if isinstance(v, str):
v = v.encode('utf-8')
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if PYTHON_VERSION >= 3:
if isinstance(body, str):
body = body.encode('utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(
self.reply, self.authid)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
self.reply.downloadProgress.connect(self.downloadProgress)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_()
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
headers = {
str(h): str(self.reply.rawHeader(h))
for h in self.reply.rawHeaderList()
}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.text) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")
except Exception as e:
raise e
finally:
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.text)
#@pyqtSlot()
def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
#self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
pass
#@pyqtSlot()
def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(
QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(k)] = str(v)
self.http_call_result.headers[str(k).lower()] = str(v)
if err != QNetworkReply.NoError:
msg = "Network error #{0}: {1}".format(self.reply.error(),
self.reply.errorString())
self.http_call_result.reason = msg
self.http_call_result.ok = False
self.msg_log(msg)
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)
elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(
msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
# since Python 3 readAll() returns a PyQt5.QByteArray, we
# want only the data
if PYTHON_VERSION >= 3:
self.http_call_result.text = self.reply.readAll().data(
).decode('utf-8')
else:
self.http_call_result.text = str(self.reply.readAll())
self.http_call_result.ok = True
self.reply.deleteLater()
#@pyqtSlot()
def sslErrors(self, reply, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v)
if self.disable_ssl_certificate_validation:
reply.ignoreSslErrors()
def getVideoLocationInfo(videoPath, dataFile=False):
""" Get basic location info about the video """
location = []
try:
if dataFile == False:
p = _spawn([
'-i', videoPath, '-ss', '00:00:00', '-to', '00:00:01', '-map',
'data-re', '-f', 'data', '-'
])
stdout_data, _ = p.communicate()
################
else:
global listOfMetadata
stdout_data = listOfMetadata[0]
################
if stdout_data == b'':
return
for packet in StreamParser(stdout_data):
if isinstance(packet, UnknownElement):
qgsu.showUserAndLogMessage(
"Error interpreting klv data, metadata cannot be read.",
"the parser did not recognize KLV data",
level=QGis.Warning)
continue
packet.MetadataList()
frameCenterLat = packet.FrameCenterLatitude
frameCenterLon = packet.FrameCenterLongitude
loc = "-"
if Reverse_geocoding_url != "":
try:
url = QUrl(
Reverse_geocoding_url.format(str(frameCenterLat),
str(frameCenterLon)))
request = QNetworkRequest(url)
reply = QgsNetworkAccessManager.instance().get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
reply.finished.disconnect(loop.quit)
loop = None
result = reply.readAll()
data = json.loads(result.data())
if "village" in data["address"] and "state" in data[
"address"]:
loc = data["address"]["village"] + \
", " + data["address"]["state"]
elif "town" in data["address"] and "state" in data[
"address"]:
loc = data["address"]["town"] + \
", " + data["address"]["state"]
else:
loc = data["display_name"]
except Exception:
qgsu.showUserAndLogMessage(
"",
"getVideoLocationInfo: failed to get address from reverse geocoding service.",
onlyLog=True)
location = [frameCenterLat, frameCenterLon, loc]
qgsu.showUserAndLogMessage(
"",
"Got Location: lon: " + str(frameCenterLon) + " lat: " +
str(frameCenterLat) + " location: " + str(loc),
onlyLog=True)
break
else:
qgsu.showUserAndLogMessage(
QCoreApplication.translate(
"QgsFmvUtils", "This video doesn't have Metadata ! : "))
except Exception as e:
qgsu.showUserAndLogMessage(
QCoreApplication.translate("QgsFmvUtils",
"Video info callback failed! : "),
str(e))
return location
class Nominatim(object):
"""Manage connexion to Nominatim."""
def __init__(self,
url="https://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QgsNetworkAccessManager.instance()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
# query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('q', query)
query_string.addQueryItem('format', 'json')
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
from XYZHubConnector.xyz_qgis.network import net_handler
from qgis.PyQt.QtCore import QEventLoop
from XYZHubConnector.xyz_qgis.models.connection import SpaceConnectionInfo
conn_info = SpaceConnectionInfo()
conn_info.set_(space_id="DicZ8XTR", token="AdOZrFlyIrXLzbAJeN5Lzts")
# minimize total number of features
# by generate tags for all levels
# smallest step for coord currently is 1 (can be smaller)
lst_tags = [format_tags(t, prefix="point") for t in precompute_tags()]
# print(lst_tags)
print("len tags", len(lst_tags))
tags_lst_obj = make_point_json(lst_tags)
print("len set tags", len(set(lst_tags)))
print("len tags_lst_obj", len(tags_lst_obj))
loop = QEventLoop()
network = NetManager(app)
total = sum(len(lst) for lst in tags_lst_obj.values())
print("total obj", total)
network.network.finished.connect(count_reply(total, loop.quit))
for i, (tags, lst_obj) in enumerate(tags_lst_obj.items()):
# print(len(lst_obj[0]["features"]), tags)
for obj in lst_obj:
pass
reply = network.add_features(conn_info, obj, tags=tags)
loop.exec_()
class Downloader(QObject):
NOT_FOUND = 0
NO_ERROR = 0
TIMEOUT_ERROR = 4
UNKNOWN_ERROR = -1
replyFinished = pyqtSignal(str, int, int)
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.queue = []
self.redirected_urls = {}
self.requestingUrls = []
self.replies = []
self.eventLoop = QEventLoop()
self.sync = False
self.fetchedFiles = {}
self.clearCounts()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.fetchTimedOut)
# network settings
self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
self.max_connection = 4
self.default_cache_expiration = 24
self.errorStatus = Downloader.NO_ERROR
def clearCounts(self):
self.fetchSuccesses = 0
self.fetchErrors = 0
self.cacheHits = 0
def fetchTimedOut(self):
self.log("Downloader.timeOut()")
self.abort()
self.errorStatus = Downloader.TIMEOUT_ERROR
def abort(self):
# clear queue and abort sent requests
self.queue = []
self.timer.stop()
for reply in self.replies:
reply.abort()
self.errorStatus = Downloader.UNKNOWN_ERROR
def replyFinishedSlot(self):
reply = self.sender()
url = reply.request().url().toString()
self.log("replyFinishedSlot: %s" % url)
if not url in self.fetchedFiles:
self.fetchedFiles[url] = None
self.requestingUrls.remove(url)
self.replies.remove(reply)
isFromCache = 0
httpStatusCode = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if reply.error() == QNetworkReply.NoError:
if httpStatusCode == 301:
new_url = str(reply.rawHeader("Location"))
self.addToQueue(new_url, url)
else:
self.fetchSuccesses += 1
if reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute):
self.cacheHits += 1
isFromCache = 1
elif not reply.hasRawHeader("Cache-Control"):
cache = QgsNetworkAccessManager.instance().cache()
if cache:
metadata = cache.metaData(reply.request().url())
# self.log("Expiration date: " + metadata.expirationDate().toString().encode("utf-8"))
if metadata.expirationDate().isNull():
metadata.setExpirationDate(
QDateTime.currentDateTime().addSecs(self.default_cache_expiration * 60 * 60))
cache.updateMetaData(metadata)
self.log(
"Default expiration date has been set: %s (%d h)" % (url, self.default_cache_expiration))
if reply.isReadable():
data = reply.readAll()
if self.redirected_urls.has_key(url):
url = self.redirected_urls[url]
self.fetchedFiles[url] = data
else:
qDebug("http status code: " + str(httpStatusCode))
# self.emit(SIGNAL('replyFinished(QString, int, int)'), url, reply.error(), isFromCache)
self.replyFinished.emit(url, reply.error(), isFromCache)
else:
if self.sync and httpStatusCode == 404:
self.fetchedFiles[url] = self.NOT_FOUND
self.fetchErrors += 1
if self.errorStatus == self.NO_ERROR:
self.errorStatus = self.UNKNOWN_ERROR
reply.deleteLater()
if debug_mode:
qDebug("queue: %d, requesting: %d" % (len(self.queue), len(self.requestingUrls)))
if len(self.queue) + len(self.requestingUrls) == 0:
# all replies have been received
if self.sync:
self.logT("eventLoop.quit()")
self.eventLoop.quit()
else:
self.timer.stop()
elif len(self.queue) > 0:
# start fetching the next file
self.fetchNext()
self.log("replyFinishedSlot End: %s" % url)
def fetchNext(self):
if len(self.queue) == 0:
return
url = self.queue.pop(0)
self.log("fetchNext: %s" % url)
request = QNetworkRequest(QUrl(url))
request.setRawHeader("User-Agent", self.userAgent)
reply = QgsNetworkAccessManager.instance().get(request)
reply.finished.connect(self.replyFinishedSlot)
self.requestingUrls.append(url)
self.replies.append(reply)
return reply
def fetchFiles(self, urlList, timeout_ms=0):
self.log("fetchFiles()")
self.sync = True
self.queue = []
self.redirected_urls = {}
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()
self.logT("eventLoop.exec_(): " + str(self.eventLoop))
self.eventLoop.exec_()
self.log("fetchFiles() End: %d" % self.errorStatus)
if timeout_ms > 0:
self.timer.stop()
return self.fetchedFiles
def addToQueue(self, url, redirected_from=None):
if url in self.queue:
return False
self.queue.append(url)
if redirected_from is not None:
self.redirected_urls[url] = redirected_from
return True
def queueCount(self):
return len(self.queue)
def finishedCount(self):
return len(self.fetchedFiles)
def unfinishedCount(self):
return len(self.queue) + len(self.requestingUrls)
def log(self, msg):
if debug_mode:
qDebug(msg)
def logT(self, msg):
if debug_mode:
qDebug("%s: %s" % (str(threading.current_thread()), msg))
def fetchFilesAsync(self, urlList, timeout_ms=0):
self.log("fetchFilesAsync()")
self.sync = False
self.queue = []
self.clearCounts()
self.errorStatus = Downloader.NO_ERROR
self.fetchedFiles = {}
if len(urlList) == 0:
return self.fetchedFiles
for url in urlList:
self.addToQueue(url)
for i in range(self.max_connection):
self.fetchNext()
if timeout_ms > 0:
self.timer.setInterval(timeout_ms)
self.timer.start()
class BaseTestAsync(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.app = start_app()
@classmethod
def tearDownClass(cls):
pass
# cls.app.quit() # useless
# stop_app() # NameError: name 'QGISAPP' is not defined
# cls.app.exitQgis() # crash when run unittest (all module)
# del cls.app
def setUp(self):
if isinstance(super(), BaseTestAsync) and hasattr(
super(), self._testMethodName):
self.skipTest("duplicated test")
# self.loop = self.app
self.loop = QEventLoop()
self._output = list()
self._idx = 0
self._lst_error = list()
self.startTime = time.time()
self._log_debug("Test start. ###################")
def tearDown(self):
self._stop_async() # useless ?
self._log_debug("Test ended. ################### \n")
self._log_info("%s: %.3fs" %
(self._id(), time.time() - self.startTime))
def _add_output(self, output):
self._output.append(output)
def output(self, idx=None):
if idx is None:
idx = self._idx
self._idx = min(idx + 1, len(self._output))
return self._output[idx] if idx < len(self._output) else None
def _stop_async(self):
self.loop.quit()
self._log_debug("Stop Async. ###################")
def _handle_error(self, e):
pretty_print_error(e)
self._stop_async()
self._lst_error.append(e)
# raise ErrorDuringTest(e)
def _process_async(self):
self.loop.processEvents()
def _wait_async(self):
t0 = time.time()
self.loop.exec_()
self._log_info("%s: wait_async end: %.3fs" %
(self._id(), time.time() - t0))
if self._lst_error:
raise AllErrorsDuringTest(self._lst_error)
def _make_async_fun(self, fun):
return AsyncFun(fun)
def _log_error(self, *a, **kw):
print(*a, file=sys.stderr, **kw)
def _log_info(self, *a, **kw):
log_truncate(*a, **kw)
def _log_debug(self, *a, **kw):
fn_name = "{}:".format(self._id())
# fn_name = sys._getframe(1).f_code.co_name
log_truncate(fn_name, *a, **kw)
def _id(self):
return self._subtest.id() if self._subtest else self.id()
def assertMultiInput(self, expected, lst_input, msg="multi input"):
for i, actual in enumerate(lst_input):
self.assertEqual(expected, actual, "{} [{}]".format(msg, i))
#unused
def assertPairEqual(self, *a):
pairs = [a[i:i + 2] for i in range(0, len(a), 2)]
return self.assertEqual(*zip(*pairs))
class NetworkAccessManager(object):
"""
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.
The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.
Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class
Usage 1 (blocking mode)
-----
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException as e:
# Handle exception
pass
Usage 2 (Non blocking mode)
-------------------------
::
NOTE! if blocking mode returns immediatly
it's up to the caller to manage listeners in case
of non blocking mode
nam = NetworkAccessManager(authcgf)
try:
nam.request('http://www.example.com', blocking=False)
nam.reply.finished.connect(a_signal_listener)
except RequestsException as e:
# Handle exception
pass
Get response using method:
nam.httpResult() that return a dictionary with keys:
'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
'content' - bytearray returned from reply
'ok' - request success [True, False]
'headers' - Dicionary containing the reply header
'reason' - fomatted message string with reply.errorString()
'exception' - the exception returne dduring execution
"""
def __init__(self, authid=None, disable_ssl_certificate_validation=False, exception_class=None, debug=False):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.exception_class = exception_class
self.on_abort = False
self.blocking_mode = False
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'content' : '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "NetworkAccessManager")
def httpResult(self):
return self.http_call_result
def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, file):
body = body.read()
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)
# necessary to trap local timout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
#@pyqtSlot()
def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
#self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
pass
#@pyqtSlot(QNetworkReply)
def requestTimedOut(self, QNetworkReply):
"""Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
# adapt http_call_result basing on receiving qgs timer timout signal
self.exception_class = RequestsExceptionTimeout
self.http_call_result.exception = RequestsExceptionTimeout("Timeout error")
#@pyqtSlot(QObject)
def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(k)] = str(v)
self.http_call_result.headers[str(k).lower()] = str(v)
if err != QNetworkReply.NoError:
# handle error
# check if errorString is empty, if so, then set err string as
# reply dump
if re.match('(.)*server replied: $', self.reply.errorString()):
errString = self.reply.errorString() + self.http_call_result.content
else:
errString = self.reply.errorString()
# check if self.http_call_result.status_code is available (client abort
# does not produce http.status_code)
if self.http_call_result.status_code:
msg = "Network error #{0}: {1}".format(
self.http_call_result.status_code, errString)
else:
msg = "Network error: {0}".format(errString)
self.http_call_result.reason = msg
self.http_call_result.ok = False
self.msg_log(msg)
# set return exception
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)
elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(msg)
elif err == QNetworkReply.OperationCanceledError:
# request abort by calling NAM.abort() => cancelled by the user
if self.on_abort:
self.http_call_result.exception = RequestsExceptionUserAbort(msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
self.http_call_result.exception = RequestsException(msg)
# overload exception to the custom exception if available
if self.exception_class:
self.http_call_result.exception = self.exception_class(msg)
else:
# Handle redirections
redirectionUrl = self.reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
if redirectionUrl is not None and redirectionUrl != self.reply.url():
if redirectionUrl.isRelative():
redirectionUrl = self.reply.url().resolved(redirectionUrl)
msg = "Redirected from '{}' to '{}'".format(
self.reply.url().toString(), redirectionUrl.toString())
self.msg_log(msg)
self.reply.deleteLater()
self.reply = None
self.request(redirectionUrl.toString())
# really end request
else:
msg = "Network success #{0}".format(self.reply.error())
self.http_call_result.reason = msg
self.msg_log(msg)
ba = self.reply.readAll()
self.http_call_result.content = bytes(ba)
self.http_call_result.ok = True
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
for k, v in list(self.http_call_result.headers.items()):
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.content) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.content)
else:
self.msg_log("Payload is > 1 KB ...")
# clean reply
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
# Disconnect all slots
self.reply.sslErrors.disconnect(self.sslErrors)
self.reply.finished.disconnect(self.replyFinished)
self.reply.downloadProgress.disconnect(self.downloadProgress)
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
#@pyqtSlot()
def sslErrors(self, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v.errorString())
if self.disable_ssl_certificate_validation:
self.reply.ignoreSslErrors()
#@pyqtSlot()
def abort(self):
"""
Handle request to cancel HTTP call
"""
if (self.reply and self.reply.isRunning()):
self.on_abort = True
self.reply.abort()
def importgeopkg(self, layer, branch, message, authorName, authorEmail,
interchange):
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
filename, layername = namesFromLayer(layer)
r = requests.get(self.url + "beginTransaction",
params={"output_format": "json"})
r.raise_for_status()
transactionId = r.json()["response"]["Transaction"]["ID"]
self._checkoutbranch(branch, transactionId)
payload = {
"authorEmail": authorEmail,
"authorName": authorName,
"message": message,
'destPath': layername,
"format": "gpkg",
"transactionId": transactionId
}
# fix_print_with_import
if interchange:
payload["interchange"] = True
filename = self.saveaudittables(filename, layername)
files = {
'fileUpload': (os.path.basename(filename), open(filename, 'rb'))
}
encoder = MultipartEncoder(files)
total = float(encoder.len)
def callback(m):
done = int(100 * m.bytes_read / total)
iface.mainWindow().statusBar().showMessage(
"Transferring geopkg to GeoGig server [{}%]".format(done))
monitor = MultipartEncoderMonitor(encoder, callback)
r = requests.post(self.url + "import.json",
params=payload,
data=monitor,
headers={'Content-Type': monitor.content_type})
self.__log(r.url, r.text, payload, "POST")
r.raise_for_status()
resp = r.json()
taskId = resp["task"]["id"]
checker = TaskChecker(self.rootUrl, taskId)
loop = QEventLoop()
checker.taskIsFinished.connect(loop.exit, Qt.QueuedConnection)
checker.start()
loop.exec_(flags=QEventLoop.ExcludeUserInputEvents)
QApplication.restoreOverrideCursor()
iface.mainWindow().statusBar().showMessage("")
if not checker.ok and "error" in checker.response["task"]:
errorMessage = checker.response["task"]["error"]["message"]
raise GeoGigException("Cannot import layer: %s" % errorMessage)
if interchange:
try:
nconflicts = checker.response["task"]["result"]["Merge"][
"conflicts"]
except KeyError, e:
nconflicts = 0
if nconflicts:
mergeCommitId = self.HEAD
importCommitId = checker.response["task"]["result"]["import"][
"importCommit"]["id"]
ancestor = checker.response["task"]["result"]["Merge"][
"ancestor"]
remote = checker.response["task"]["result"]["Merge"]["ours"]
try:
featureIds = checker.response["task"]["result"]["import"][
"NewFeatures"]["type"][0].get("ids", [])
except:
featureIds = []
con = sqlite3.connect(filename)
cursor = con.cursor()
geomField = cursor.execute(
"SELECT column_name FROM gpkg_geometry_columns WHERE table_name='%s';"
% layername).fetchone()[0]
def _local(fid):
cursor.execute(
"SELECT gpkg_fid FROM %s_fids WHERE geogig_fid='%s';" %
(layername, fid))
gpkgfid = int(cursor.fetchone()[0])
request = QgsFeatureRequest()
request.setFilterFid(gpkgfid)
try:
feature = next(layer.getFeatures(request))
except:
return None
def _ensureNone(v):
if v == NULL:
return None
else:
return v
local = {
f.name(): _ensureNone(feature[f.name()])
for f in layer.pendingFields()
}
try:
local[geomField] = feature.geometry().exportToWkt()
except:
local[geomField] = None
return local
conflicts = []
conflictsResponse = _ensurelist(
checker.response["task"]["result"]["Merge"]["Feature"])
for c in conflictsResponse:
if c["change"] == "CONFLICT":
remoteFeatureId = c["ourvalue"]
localFeatureId = c["theirvalue"]
localFeature = _local(c["id"].split("/")[-1])
conflicts.append(
ConflictDiff(self, c["id"], ancestor, remote,
importCommitId, localFeature,
localFeatureId, remoteFeatureId,
transactionId))
cursor.close()
con.close()
else:
#self._checkoutbranch("master", transactionId)
self.closeTransaction(transactionId)
mergeCommitId = checker.response["task"]["result"][
"newCommit"]["id"]
importCommitId = checker.response["task"]["result"][
"importCommit"]["id"]
try:
featureIds = checker.response["task"]["result"][
"NewFeatures"]["type"][0].get("id", [])
except:
featureIds = []
conflicts = []
featureIds = [(f["provided"], f["assigned"]) for f in featureIds]
return mergeCommitId, importCommitId, conflicts, featureIds
class ConnexionOAPI(object):
"""
Manage connexion to the overpass API
"""
def __init__(self, url="http://overpass-api.de/api/", output=None):
"""
Constructor
@param url:URL of OverPass
@type url:str
@param output:Output desired (XML or JSON)
@type output:str
"""
if not url:
url = "http://overpass-api.de/api/"
self.__url = url
self.result_path = None
if output not in (None, "json", "xml"):
raise OutPutFormatException
self.__output = output
self.network = QgsNetworkAccessManager.instance()
self.network_reply = None
self.loop = None
def query(self, query):
"""
Make a query to the overpass
@param query:Query to execute
@type query:str
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url + 'interpreter')
# The output format can be forced (JSON or XML)
if self.__output:
query = re.sub(r'output="[a-z]*"',
'output="' + self.__output + '"', query)
query = re.sub(r'\[out:[a-z]*', '[out:' + self.__output, query)
# noinspection PyCallByClass
# encoded_query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('data', query)
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
lines = lines[-10:] # Get last 10 lines
timeout = '<remark> runtime error: Query timed out in "[a-z]+" ' \
'at line [\d]+ after ([\d]+) seconds. </remark>'
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
else:
return self.result_path
elif self.network_reply.error() == QNetworkReply.UnknownContentError:
raise OverpassBadRequestException
else:
raise NetWorkErrorException(suffix="Overpass OSM API")
def _end_of_request(self):
tf = QTemporaryFile(os.path.join(QDir.tempPath(),
'request-XXXXXX.osm'))
tf.setAutoRemove(False)
tf.open(QIODevice.WriteOnly | QIODevice.Text)
tf.write(self.network_reply.readAll().simplified())
tf.close()
self.result_path = tf.fileName()
self.loop.quit()
class AsyncNetworkAccessManager(object):
"""
This class is an Async version of NetworkAccessManager trying to mimicks
QgsNetworkAccessManager insteqad of blocking httplib2.
The main reason of using AsyncNetworkAccessManager respect
QgsNetworkAccessManager is that this class manage AuthId in the constructor
and offer wrapper for updateNetworkReply used to update SSH reply basing
on authid stored credentials in QGIS AuthDB
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.
The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.
Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class
Usage 1 (bloking mode)
----------------------
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException, e:
# Handle exception
pass
Usage 2 (No bloking mode)
-------------------------
::
nam = NetworkAccessManager(authcgf)
try:
nam.request('http://www.example.com', blocking=False)
nam.reply.finished.connect(a_signal_listener)
except RequestsException, e:
# Handle exception
pass
"""
def __init__(self, authid=None, disable_ssl_certificate_validation=False, exception_class=None, debug=True):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.onAbort = False
self.blockingMode = False
self.exception_class = exception_class
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'text' : '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "AsyncNetworkAccessManager")
def httpResult(self):
return self.http_call_result
def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blockingMode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.onAbort = False
self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, file):
body = body.read()
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(self.reply, self.authid)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
# Call
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediately
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blockingMode:
return (None, None)
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
finally:
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
# Disconnect all slots
self.reply.sslErrors.disconnect(self.sslErrors)
self.reply.finished.disconnect(self.replyFinished)
self.reply.finished.disconnect(self.el.quit)
self.reply.downloadProgress.disconnect(self.downloadProgress)
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.text)
@pyqtSlot()
def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
@pyqtSlot()
def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
# get result
ba = self.reply.readAll()
self.http_call_result.text = bytes(ba)
# dump header
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(k)] = str(v)
self.http_call_result.headers[str(k).lower()] = str(v)
if err != QNetworkReply.NoError:
# check if errorString is empty, if so, then set err string as
# reply dump
if re.match('(.)*server replied: $', self.reply.errorString()):
errString = self.reply.errorString() + self.http_call_result.text
else:
errString = self.reply.errorString()
msg = "Network error #{0}: {1}".format(
self.http_call_result.status_code, errString)
self.http_call_result.reason = msg
self.http_call_result.ok = False
self.msg_log(msg)
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)
elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(msg)
elif err == QNetworkReply.OperationCanceledError:
# request abort by calling ANAM.abort() => cancelling by the user
if self.onAbort:
self.http_call_result.exception = RequestsExceptionUserAbort(msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
msg = "Network success #{0}".format(self.reply.error())
self.http_call_result.reason = msg
self.msg_log(msg)
self.http_call_result.ok = True
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
for k, v in list(self.http_call_result.headers.items()):
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.text) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")
@pyqtSlot()
def sslErrors(self, reply, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v)
if self.disable_ssl_certificate_validation:
reply.ignoreSslErrors()
@pyqtSlot()
def abort(self):
"""
Handle request to cancel HTTP call
"""
if (self.reply and self.reply.isRunning()):
self.onAbort = True
self.reply.abort()
class TestPyQgsWFSProviderGUI(unittest.TestCase):
@classmethod
def setUpClass(cls):
"""Run before all tests"""
QCoreApplication.setOrganizationName("QGIS_Test")
QCoreApplication.setOrganizationDomain("QGIS_TestPyQgsWFSProviderGUI.com")
QCoreApplication.setApplicationName("QGIS_TestPyQgsWFSProviderGUI")
QSettings().clear()
start_app()
cls.basetestpath = tempfile.mkdtemp().replace('\\', '/')
@classmethod
def tearDownClass(cls):
"""Run after all tests"""
QSettings().clear()
if cls.basetestpath is not None:
shutil.rmtree(cls.basetestpath, True)
def get_button(self, main_dialog, text):
buttonBox = main_dialog.findChild(QDialogButtonBox, "buttonBox")
self.assertIsNotNone(buttonBox)
button = None
for button in buttonBox.buttons():
if str(button.text()) == text:
return button
self.assertIsNotNone(None)
return None
def get_button_add(self, main_dialog):
return self.get_button(main_dialog, "&Add")
def get_button_build_query(self, main_dialog):
return self.get_button(main_dialog, "&Build query")
def wait_object_destruction(self, my_object):
loop = QEventLoop()
name = my_object.objectName()
my_object.destroyed.connect(loop.quit)
loop.exec_()
self.assertIsNone(find_window(name))
return None
def test(self):
# This test is quite fragile as it depends on windows manager behaviour
# regarding focus, so not surprising it doesn't pass
# on other platforms than Linux.
#if 'TRAVIS_OS_NAME' in os.environ and os.environ['TRAVIS_OS_NAME'] == 'osx':
# return
main_dialog = QgsProviderRegistry.instance().selectWidget("WFS")
main_dialog.setProperty("hideDialogs", True)
self.assertIsNotNone(main_dialog)
# Create new connection
btnNew = main_dialog.findChild(QWidget, "btnNew")
self.assertIsNotNone(btnNew)
QTest.mouseClick(btnNew, Qt.LeftButton)
new_conn = find_window('QgsNewHttpConnectionBase')
self.assertIsNotNone(new_conn)
txtName = new_conn.findChild(QLineEdit, "txtName")
self.assertIsNotNone(txtName)
txtName.setText("test_connection")
txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
self.assertIsNotNone(txtUrl)
txtUrl.setText("test_url")
new_conn.accept()
# Wait for object to be destroyed
new_conn = self.wait_object_destruction(new_conn)
# Try to connect
btnConnect = main_dialog.findChild(QWidget, "btnConnect")
self.assertIsNotNone(btnConnect)
QTest.mouseClick(btnConnect, Qt.LeftButton)
# Depends on asynchronous signal
QApplication.processEvents()
error_box = find_window('WFSCapabilitiesErrorBox')
self.assertIsNotNone(error_box)
# Close error box
error_box.accept()
# Wait for object to be destroyed
error_box = self.wait_object_destruction(error_box)
# Edit connection
btnEdit = main_dialog.findChild(QWidget, "btnEdit")
self.assertIsNotNone(btnEdit)
QTest.mouseClick(btnEdit, Qt.LeftButton)
new_conn = find_window('QgsNewHttpConnectionBase',)
self.assertIsNotNone(new_conn)
txtName = new_conn.findChild(QLineEdit, "txtName")
self.assertIsNotNone(txtName)
txtName.setText("test_connection")
txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
self.assertIsNotNone(txtUrl)
endpoint = self.basetestpath + '/fake_qgis_http_endpoint'
expected_endpoint = endpoint
if sys.platform == 'win32' and expected_endpoint[1] == ':':
expected_endpoint = expected_endpoint[0] + expected_endpoint[2:]
with open(sanitize(endpoint, '?SERVICE=WFS?REQUEST=GetCapabilities?ACCEPTVERSIONS=2.0.0,1.1.0,1.0.0'), 'wb') as f:
f.write("""
<wfs:WFS_Capabilities version="2.0.0" xmlns="http://www.opengis.net/wfs/2.0" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://schemas.opengis.net/gml/3.2" xmlns:fes="http://www.opengis.net/fes/2.0">
<FeatureTypeList>
<FeatureType>
<Name>my:typename</Name>
<Title>Title</Title>
<Abstract>Abstract</Abstract>
<DefaultCRS>urn:ogc:def:crs:EPSG::4326</DefaultCRS>
<ows:WGS84BoundingBox>
<ows:LowerCorner>-71.123 66.33</ows:LowerCorner>
<ows:UpperCorner>-65.32 78.3</ows:UpperCorner>
</ows:WGS84BoundingBox>
</FeatureType>
</FeatureTypeList>
<fes:Filter_Capabilities>
<fes:Spatial_Capabilities>
<fes:GeometryOperands>
<fes:GeometryOperand name="gml:Envelope"/>
<fes:GeometryOperand name="gml:Point"/>
<fes:GeometryOperand name="gml:MultiPoint"/>
<fes:GeometryOperand name="gml:LineString"/>
<fes:GeometryOperand name="gml:MultiLineString"/>
<fes:GeometryOperand name="gml:Polygon"/>
<fes:GeometryOperand name="gml:MultiPolygon"/>
<fes:GeometryOperand name="gml:MultiGeometry"/>
</fes:GeometryOperands>
<fes:SpatialOperators>
<fes:SpatialOperator name="Disjoint"/>
<fes:SpatialOperator name="Equals"/>
<fes:SpatialOperator name="DWithin"/>
<fes:SpatialOperator name="Beyond"/>
<fes:SpatialOperator name="Intersects"/>
<fes:SpatialOperator name="Touches"/>
<fes:SpatialOperator name="Crosses"/>
<fes:SpatialOperator name="Within"/>
<fes:SpatialOperator name="Contains"/>
<fes:SpatialOperator name="Overlaps"/>
<fes:SpatialOperator name="BBOX"/>
</fes:SpatialOperators>
</fes:Spatial_Capabilities>
<fes:Functions>
<fes:Function name="abs">
<fes:Returns>xs:int</fes:Returns>
<fes:Arguments>
<fes:Argument name="param">
<fes:Type>xs:int</fes:Type>
</fes:Argument>
</fes:Arguments>
</fes:Function>
</fes:Functions>
</fes:Filter_Capabilities>
</wfs:WFS_Capabilities>""".encode('UTF-8'))
txtUrl.setText("http://" + endpoint)
new_conn.accept()
# Wait for object to be destroyed
new_conn = self.wait_object_destruction(new_conn)
# Try to connect
btnConnect = main_dialog.findChild(QWidget, "btnConnect")
self.assertIsNotNone(btnConnect)
QTest.mouseClick(btnConnect, Qt.LeftButton)
# We need to show (and raise for Mac) the dialog so that the focus changes
self.loop = QEventLoop()
treeView = main_dialog.findChild(QTreeView, "treeView")
treeView.selectionModel().currentRowChanged.connect(main_dialog.hide)
treeView.selectionModel().currentRowChanged.connect(self.loop.quit)
main_dialog.show()
main_dialog.raise_()
self.loop.exec_()
# Add layer
buttonAdd = self.get_button_add(main_dialog)
self.assertTrue(buttonAdd.isEnabled())
self.addWfsLayer_uri = None
self.addWfsLayer_layer_name = None
main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
QTest.mouseClick(buttonAdd, Qt.LeftButton)
self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=')
self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
# Click on Build Query
buttonBuildQuery = self.get_button_build_query(main_dialog)
self.assertTrue(buttonBuildQuery.isEnabled())
QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
error_box = find_window('WFSFeatureTypeErrorBox')
self.assertIsNotNone(error_box)
# Close error box
error_box.accept()
# Wait for object to be destroyed
error_box = self.wait_object_destruction(error_box)
# Click again but with valid DescribeFeatureType
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAME=my:typename'), 'wb') as f:
f.write("""
<xsd:schema xmlns:my="http://my" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://my">
<xsd:import namespace="http://www.opengis.net/gml/3.2"/>
<xsd:complexType name="typenameType">
<xsd:complexContent>
<xsd:extension base="gml:AbstractFeatureType">
<xsd:sequence>
<xsd:element maxOccurs="1" minOccurs="0" name="intfield" nillable="true" type="xsd:int"/>
<xsd:element maxOccurs="1" minOccurs="0" name="geometryProperty" nillable="true" type="gml:PolygonPropertyType"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:element name="typename" substitutionGroup="gml:_Feature" type="my:typenameType"/>
</xsd:schema>
""".encode('UTF-8'))
QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
# Check that the combos are properly initialized
dialog = find_window('QgsSQLComposerDialogBase')
self.assertIsNotNone(dialog)
mTablesCombo = dialog.findChild(QComboBox, "mTablesCombo")
self.assertIsNotNone(mTablesCombo)
self.assertEqual(mTablesCombo.itemText(1), 'typename (Title)')
mColumnsCombo = dialog.findChild(QComboBox, "mColumnsCombo")
self.assertIsNotNone(mColumnsCombo)
self.assertEqual(mColumnsCombo.itemText(1), 'intfield (int)')
self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 2), 'geometryProperty (geometry)')
self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 1), '*')
mFunctionsCombo = dialog.findChild(QComboBox, "mFunctionsCombo")
self.assertIsNotNone(mFunctionsCombo)
self.assertEqual(mFunctionsCombo.itemText(1), 'abs(param: int): int')
mSpatialPredicatesCombo = dialog.findChild(QComboBox, "mSpatialPredicatesCombo")
self.assertIsNotNone(mSpatialPredicatesCombo)
self.assertEqual(mSpatialPredicatesCombo.itemText(1), 'ST_Disjoint(geometry, geometry): boolean')
mWhereEditor = dialog.findChild(QTextEdit, "mWhereEditor")
self.assertIsNotNone(mWhereEditor)
mWhereEditor.setText('1 = 1')
dialog.accept()
# Wait for object to be destroyed
dialog = self.wait_object_destruction(dialog)
# Add layer
buttonAdd = self.get_button_add(main_dialog)
self.assertTrue(buttonAdd.isEnabled())
self.addWfsLayer_uri = None
self.addWfsLayer_layer_name = None
main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
QTest.mouseClick(buttonAdd, Qt.LeftButton)
self.assertEqual(self.addWfsLayer_uri, ' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' table="" sql=SELECT * FROM typename WHERE 1 = 1')
self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
#main_dialog.setProperty("hideDialogs", None)
#main_dialog.exec_()
def slotAddWfsLayer(self, uri, layer_name):
self.addWfsLayer_uri = uri
self.addWfsLayer_layer_name = layer_name
class Nominatim:
"""Manage connexion to Nominatim."""
def __init__(self,
url="https://nominatim.openstreetmap.org/search?format=json"):
"""
Constructor
@param url:URL of Nominatim
@type url:str
"""
self.__url = url
self.network = QgsNetworkAccessManager.instance()
self.data = None
self.network_reply = None
self.loop = None
def query(self, query):
"""
Perform a nominatim query
@param query: Query to execute
@type query: str
@raise NetWorkErrorException
@return: the result of the query
@rtype: str
"""
url_query = QUrl(self.__url)
# query = QUrl.toPercentEncoding(query)
query_string = QUrlQuery()
query_string.addQueryItem('q', query)
query_string.addQueryItem('format', 'json')
query_string.addQueryItem('info', 'QgisQuickOSMPlugin')
url_query.setQuery(query_string)
request = QNetworkRequest(url_query)
# request.setRawHeader("User-Agent", "QuickOSM")
self.network_reply = self.network.get(request)
self.loop = QEventLoop()
self.network.finished.connect(self._end_of_request)
self.loop.exec_()
if self.network_reply.error() == QNetworkReply.NoError:
return json.loads(self.data)
else:
raise NetWorkErrorException(suffix="Nominatim API")
def _end_of_request(self):
self.data = self.network_reply.readAll().data().decode('utf-8')
self.loop.quit()
def get_first_polygon_from_query(self, query):
"""
Get first OSM_ID of a Nominatim area
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "relation":
return result['osm_id']
# If no result has been return
raise NominatimAreaException
def get_first_point_from_query(self, query):
"""
Get first longitude, latitude of a Nominatim point
@param query: Query to execute
@type query: str
@raise NominatimAreaException:
@return: First relation's osm_id
@rtype: str
"""
data = self.query(query)
for result in data:
if result['osm_type'] == "node":
return result['lon'], result['lat']
# If no result has been return
raise NominatimAreaException
class NetworkAccessManager(object):
"""
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.
The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.
Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class
Usage 1 (blocking mode)
-----
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException as e:
# Handle exception
pass
Usage 2 (Non blocking mode)
-------------------------
::
NOTE! if blocking mode returns immediatly
it's up to the caller to manage listeners in case
of non blocking mode
nam = NetworkAccessManager(authcgf)
try:
nam.request('http://www.example.com', blocking=False)
nam.reply.finished.connect(a_signal_listener)
except RequestsException as e:
# Handle exception
pass
Get response using method:
nam.httpResult() that return a dictionary with keys:
'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
'content' - bytearray returned from reply
'ok' - request success [True, False]
'headers' - Dicionary containing the reply header
'reason' - fomatted message string with reply.errorString()
'exception' - the exception returne dduring execution
"""
def __init__(self,
authid=None,
disable_ssl_certificate_validation=False,
exception_class=None,
debug=True,
timeout=None):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.exception_class = exception_class
self.on_abort = False
self.blocking_mode = False
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'content': '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
if timeout is None:
timeout = 60
self.timeout = timeout
def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "NetworkAccessManager")
def httpResult(self):
return self.http_call_result
def auth_manager(self):
return QgsApplication.authManager()
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None,
blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
if k and v:
req.setRawHeader(k.encode(), v.encode())
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, io.IOBase):
body = body.read()
if isinstance(body, str):
body = body.encode()
if isinstance(body, dict):
body = str(json.dumps(body)).encode(encoding='utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkReply(self.reply, self.authid)
QgsNetworkAccessManager.instance().setTimeout(int(self.timeout) * 1000)
# necessary to trap local timeout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(
self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
#self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
pass
def requestTimedOut(self, reply):
"""Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
# adapt http_call_result basing on receiving qgs timer timout signal
self.exception_class = RequestsExceptionTimeout
self.http_call_result.exception = RequestsExceptionTimeout(
"Timeout error")
def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(
QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(
k.data(), encoding='utf-8')] = str(v.data(), encoding='utf-8')
self.http_call_result.headers[str(
k.data(), encoding='utf-8').lower()] = str(v.data(),
encoding='utf-8')
if err != QNetworkReply.NoError:
# handle error
# check if errorString is empty, if so, then set err string as
# reply dump
if re.match('(.)*server replied: $', self.reply.errorString()):
errString = self.reply.errorString(
) + self.http_call_result.content
else:
errString = self.reply.errorString()
# check if self.http_call_result.status_code is available (client abort
# does not produce http.status_code)
if self.http_call_result.status_code:
msg = "Network error #{0}: {1}".format(
self.http_call_result.status_code, errString)
else:
msg = "Network error: {0}".format(errString)
self.http_call_result.reason = msg
self.http_call_result.text = str(self.reply.readAll().data(),
encoding='utf-8')
self.http_call_result.ok = False
self.msg_log(msg)
# set return exception
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)
elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(
msg)
elif err == QNetworkReply.OperationCanceledError:
# request abort by calling NAM.abort() => cancelled by the user
if self.on_abort:
self.http_call_result.exception = RequestsExceptionUserAbort(
msg)
else:
self.http_call_result.exception = RequestsException(msg)
else:
self.http_call_result.exception = RequestsException(msg)
# overload exception to the custom exception if available
if self.exception_class:
self.http_call_result.exception = self.exception_class(msg)
else:
# Handle redirections
redirectionUrl = self.reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if redirectionUrl is not None and redirectionUrl != self.reply.url(
):
if redirectionUrl.isRelative():
redirectionUrl = self.reply.url().resolved(redirectionUrl)
msg = "Redirected from '{}' to '{}'".format(
self.reply.url().toString(), redirectionUrl.toString())
self.msg_log(msg)
self.reply.deleteLater()
self.reply = None
self.request(redirectionUrl.toString())
# really end request
else:
msg = "Network success #{0}".format(self.reply.error())
self.http_call_result.reason = msg
self.msg_log(msg)
ba = self.reply.readAll()
self.http_call_result.content = bytes(ba)
self.http_call_result.text = str(ba.data(), encoding='utf-8')
self.http_call_result.ok = True
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
for k, v in list(self.http_call_result.headers.items()):
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.content) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")
# clean reply
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
# Disconnect all slots
self.reply.sslErrors.disconnect(self.sslErrors)
self.reply.finished.disconnect(self.replyFinished)
self.reply.downloadProgress.disconnect(self.downloadProgress)
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
def sslErrors(self, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v.errorString())
if self.disable_ssl_certificate_validation:
self.reply.ignoreSslErrors()
def abort(self):
"""
Handle request to cancel HTTP call
"""
if (self.reply and self.reply.isRunning()):
self.on_abort = True
self.reply.abort()
def upload_files(self, layer, field_index, features):
"""
Upload given features' source files to remote server and return a dict
formatted as changeAttributeValues expects to update 'datos' attribute
to a remote location.
"""
if not QSettings().value(
'Asistente-LADM_COL/sources/document_repository', False, bool):
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"The source files were not uploaded to the document repository because you have that option unchecked. You can still upload the source files later using the 'Upload Pending Source Files' menu."
), Qgis.Info, 10)
return dict()
# Test if we have Internet connection and a valid service
dlg = self.qgis_utils.get_settings_dialog()
res, msg = dlg.is_source_service_valid()
if not res:
msg['text'] = QCoreApplication.translate(
"SourceHandler",
"No file could be uploaded to the document repository. You can do it later from the 'Upload Pending Source Files' menu. Reason: {}"
).format(msg['text'])
self.message_with_duration_emitted.emit(
msg['text'], Qgis.Info,
20) # The data is still saved, so always show Info msg
return dict()
file_features = [
feature for feature in features if not feature[field_index] == NULL
and os.path.isfile(feature[field_index])
]
total = len(features)
not_found = total - len(file_features)
upload_dialog = UploadProgressDialog(len(file_features), not_found)
upload_dialog.show()
count = 0
upload_errors = 0
new_values = dict()
for feature in file_features:
data_url = feature[field_index]
file_name = os.path.basename(data_url)
nam = QNetworkAccessManager()
#reply.downloadProgress.connect(upload_dialog.update_current_progress)
multiPart = QHttpMultiPart(QHttpMultiPart.FormDataType)
textPart = QHttpPart()
textPart.setHeader(QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"driver\""))
textPart.setBody(QByteArray().append('Local'))
filePart = QHttpPart()
filePart.setHeader(
QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"file\"; filename=\"{}\"".format(
file_name)))
file = QFile(data_url)
file.open(QIODevice.ReadOnly)
filePart.setBodyDevice(file)
file.setParent(
multiPart
) # we cannot delete the file now, so delete it with the multiPart
multiPart.append(filePart)
multiPart.append(textPart)
service_url = '/'.join([
QSettings().value(
'Asistente-LADM_COL/sources/service_endpoint',
DEFAULT_ENDPOINT_SOURCE_SERVICE),
SOURCE_SERVICE_UPLOAD_SUFFIX
])
request = QNetworkRequest(QUrl(service_url))
reply = nam.post(request, multiPart)
#reply.uploadProgress.connect(upload_dialog.update_current_progress)
reply.error.connect(self.error_returned)
multiPart.setParent(reply)
# We'll block execution until we get response from the server
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
response = reply.readAll()
data = QTextStream(response, QIODevice.ReadOnly)
content = data.readAll()
if content is None:
self.log.logMessage(
"There was an error uploading file '{}'".format(data_url),
PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
try:
response = json.loads(content)
except json.decoder.JSONDecodeError:
self.log.logMessage(
"Couldn't parse JSON response from server for file '{}'!!!"
.format(data_url), PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
if 'error' in response:
self.log.logMessage(
"STATUS: {}. ERROR: {} MESSAGE: {} FILE: {}".format(
response['status'], response['error'],
response['message'], data_url), PLUGIN_NAME,
Qgis.Critical)
upload_errors += 1
continue
reply.deleteLater()
if 'url' not in response:
self.log.logMessage(
"'url' attribute not found in JSON response for file '{}'!"
.format(data_url), PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
url = self.get_file_url(response['url'])
new_values[feature.id()] = {field_index: url}
count += 1
upload_dialog.update_total_progress(count)
if not_found > 0:
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} records {} not uploaded to the document repository because {} file path is NULL or it couldn't be found in the local disk!"
).format(
not_found, total,
QCoreApplication.translate("SourceHandler", "was")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "their")), Qgis.Info, 0)
if len(new_values):
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files {} uploaded to the document repository and {} remote location stored in the database!"
).format(
len(new_values), total,
QCoreApplication.translate("SourceHandler", "was")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "their")), Qgis.Info, 0)
if upload_errors:
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files could not be uploaded to the document repository because of upload errors! See log for details."
).format(upload_errors, total), Qgis.Info, 0)
return new_values
