def testInvalidAuthFileDownload(self):
"""
Download a protected map tile without authcfg
"""
qs = "?" + "&".join([
"%s=%s" % i
for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())
])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, None, False)
downloader.downloadCompleted.connect(
partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot,
'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot,
'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
self.assertTrue(self.error_was_called)
self.assertTrue(
"Download failed: Host requires authentication"
in str(self.error_args),
"Error args is: %s" % str(self.error_args))
def download_image(self, url):
res = False
img = None
msg = {'text': '', 'level': Qgis.Warning}
if url:
self.logger.info(__name__, "Downloading file from {}".format(url))
msg = "Downloading image from document repository (this might take a while)..."
with ProcessWithStatus(msg):
if self.qgis_utils.is_connected(TEST_SERVER):
nam = QNetworkAccessManager()
request = QNetworkRequest(QUrl(url))
reply = nam.get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
status = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if status == 200:
res = True
img = reply.readAll()
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to the server. The server might be down or the service cannot be reached at the given URL."
)
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to Internet.")
else:
res = False
msg['text'] = QCoreApplication.translate("SettingsDialog",
"Not valid URL")
if not res:
self.logger.log_message(__name__, msg['text'], msg['level'])
return (res, img)
def test_api_usage_async(self):
"""Test standard async API usage"""
api = NzElectoralApi('http://localhost:%s' % self.port)
nam = api.status()
self.last_result = ''
expected = {
"version": self.API_VERSION,
"gmsVersion": "LINZ_Output_20180108_2018_V1_00"
}
el = QEventLoop()
def f(nam):
"""Wrapper"""
self.last_result = api.parse_async(nam)['content']
nam.reply.finished.connect(partial(f, nam))
nam.reply.finished.connect(el.quit)
el.exec_(QEventLoop.ExcludeUserInputEvents)
self.assertEqual(self.last_result, expected)
def _post_sync(self,
qurl: QUrl,
timeout: int = 20000,
data: bytes = b'',
content_type=None):
'''
synchronous POST-request
'''
request = QNetworkRequest(qurl)
if content_type:
request.setHeader(QNetworkRequest.ContentTypeHeader, content_type)
# newer versions of QGIS (3.6+) support synchronous requests
if hasattr(self._manager, 'blockingPost'):
reply = self._manager.blockingPost(request,
data,
forceRefresh=True)
# use blocking event loop for older versions
else:
loop = QEventLoop()
timer = QTimer()
timer.setSingleShot(True)
# reply or timeout break event loop, whoever comes first
timer.timeout.connect(loop.quit)
reply = self._manager.post(request, data)
reply.finished.connect(loop.quit)
timer.start(timeout)
# start blocking loop
loop.exec()
loop.deleteLater()
if not timer.isActive():
reply.deleteLater()
raise ConnectionError('Timeout')
timer.stop()
if reply.error():
self.error.emit(reply.errorString())
raise ConnectionError(reply.errorString())
res = Reply(reply)
self.finished.emit(res)
return res
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language", b"fr")
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
raise OverpassTimeoutException
# The download is done, checking for not complete OSM file.
# Overpass might aborted the request with HTTP 200.
file_obj = codecs.open(self.result_path, 'r', 'utf-8')
file_obj.seek(0, 2)
fsize = file_obj.tell()
file_obj.seek(max(fsize - 1024, 0), 0)
lines = file_obj.readlines()
file_obj.close()
lines = lines[-10:] # Get last 10 lines
timeout = (
'<remark> runtime error: Query timed out in "[a-z]+" at line '
'[\d]+ after ([\d]+) seconds. </remark>')
if re.search(timeout, ''.join(lines)):
raise OverpassTimeoutException
# Everything went fine
return self.result_path
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.queue = []
self.redirected_urls = {}
self.requestingUrls = []
self.replies = []
self.eventLoop = QEventLoop()
self.sync = False
self.fetchedFiles = {}
self.clearCounts()
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.timeout.connect(self.fetchTimedOut)
# network settings
self.userAgent = "QuickMapServices tile layer (+https://github.com/nextgis/quickmapservices)"
self.max_connection = 4
self.default_cache_expiration = 24
self.errorStatus = Downloader.NO_ERROR
def get_available_providers(providers_uri):
"""Fetch the list of available providers from BCS endpoint,
apparently this API method does not require auth"""
# For testing purposes, we can also access to a json file directly
if not providers_uri.startswith('http'):
try:
j = json.load(open(providers_uri, encoding='utf-8'))
except:
j = json.load(open(providers_uri))
else:
t = mktemp()
q = QgsFileDownloader(QUrl(providers_uri), t)
loop = QEventLoop()
q.downloadExited.connect(loop.quit)
loop.exec_()
if not os.path.isfile(t):
return []
with open(t) as f:
j = json.load(f)
os.unlink(t)
return j
def process_delete_call(self, url, report_url=True):
"""
Run a DELETE request and return reply data
:param url: url for request
:param report_url: True if URL should be reported to feedback
:return: response or error message in json format
"""
if self.connection.read_only:
return {
"error": {
"msg": "Graphium connection is set to read-only!"
}
}
url_query = QUrl(url)
if report_url:
self.report_info('DELETE ' + url_query.toString())
request = QNetworkRequest(url_query)
if self.connection.auth_cfg != '':
self.auth = 0
config = QgsAuthMethodConfig()
QgsApplication.authManager().loadAuthenticationConfig(
self.connection.auth_cfg, config, True)
concatenated = config.configMap(
)['username'] + ":" + config.configMap()['password']
data = base64.encodebytes(concatenated.encode("utf-8")).replace(
'\n'.encode("utf-8"), ''.encode("utf-8"))
request.setRawHeader("Authorization".encode("utf-8"),
("Basic %s" % data).encode("utf-8"))
request.setRawHeader("Accept".encode("utf-8"),
"*/*".encode("utf-8"))
loop = QEventLoop()
reply = self.network_access_manager.deleteResource(request)
reply.finished.connect(loop.quit)
loop.exec_()
return self.process_q_reply(reply)
def _make_download(self, url, destination, cancel=False):
self.completed_was_called = False
self.error_was_called = False
self.canceled_was_called = False
self.progress_was_called = False
self.exited_was_called = False
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
if cancel:
downloader.downloadProgress.connect(downloader.cancelDownload)
loop.exec_()
def run(self):
"""Run the query.
@raise OverpassBadRequestException,NetWorkErrorException,
OverpassTimeoutException
@return: The result of the query.
@rtype: str
"""
loop = QEventLoop()
downloader = QgsFileDownloader(self._url,
self.result_path,
delayStart=True)
downloader.downloadExited.connect(loop.quit)
downloader.downloadError.connect(self.error)
downloader.downloadCanceled.connect(self.canceled)
downloader.downloadCompleted.connect(self.completed)
downloader.startDownload()
loop.exec_()
for message in self.errors:
self.is_query_timed_out(message)
self.is_bad_request(message)
LOGGER.error(message)
if len(self.errors):
raise NetWorkErrorException('Overpass API', ', '.join(self.errors))
osm_file = QFileInfo(self.result_path)
if not osm_file.exists() and not osm_file.isFile():
# Do not raise a QuickOSM exception here
# It must be a bug from QuickOSM
raise FileNotFoundError
self.check_file(self.result_path)
# Everything went fine
return self.result_path
def testValidAuthFileDownload(self):
"""
Download a map tile with valid authcfg
"""
qs = "?" + "&".join(["%s=%s" % i for i in list({
"MAP": urllib.parse.quote(self.project_path),
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetMap",
"LAYERS": "testlayer_èé".replace('_', '%20'),
"STYLES": "",
"FORMAT": "image/png",
"BBOX": "-16817707,-4710778,5696513,14587125",
"HEIGHT": "500",
"WIDTH": "500",
"CRS": "EPSG:3857"
}.items())])
url = '%s://%s:%s/%s' % (self.protocol, self.hostname, self.port, qs)
destination = tempfile.mktemp()
loop = QEventLoop()
downloader = QgsFileDownloader(QUrl(url), destination, self.auth_config.id(), False)
downloader.downloadCompleted.connect(partial(self._set_slot, 'completed'))
downloader.downloadExited.connect(partial(self._set_slot, 'exited'))
downloader.downloadCanceled.connect(partial(self._set_slot, 'canceled'))
downloader.downloadError.connect(partial(self._set_slot, 'error'))
downloader.downloadProgress.connect(partial(self._set_slot, 'progress'))
downloader.downloadExited.connect(loop.quit)
loop.exec_()
# Check the we've got a likely PNG image
self.assertTrue(self.completed_was_called)
self.assertTrue(os.path.getsize(destination) > 2000, "Image size: %s" % os.path.getsize(destination)) # > 1MB
with open(destination, 'rb') as f:
self.assertTrue(b'PNG' in f.read()) # is a PNG
def loadModel(self):
imd = None
try:
loader = ModelLoader(self.ui.mDataLineEdit.text())
models = loader.detect_models()
model_names = map(lambda m: m.name, models)
self._log_output("Looking up models: " + ', '.join(model_names))
ili = loader.gen_lookup_ili()
qDebug(ili)
wpsreq = self._create_wps_request(ili)
url = self.ui.mIlisMetaUrlLineEdit.text()
req = QNetworkRequest(QUrl(url))
req.setHeader(QNetworkRequest.ContentTypeHeader, 'application/xml')
reply = QgsNetworkAccessManager.instance().post(req, wpsreq)
# Wait for reply or timeout
loop = QEventLoop()
reply.finished.connect(loop.quit)
QTimer.singleShot(15000, reply.abort)
loop.exec_()
if reply.isFinished() and reply.error() == QNetworkReply.NoError:
result = reply.readAll()
imd = self._parse_wps_response(result)
except:
qDebug("Exception during IlisModel download")
if imd is None:
self._show_log_window()
QgsApplication.messageLog().logMessage(
"Couldn't download Ilismeta model", "Interlis",
Qgis.MessageLevel(1))
self.ui.mModelLineEdit.setText("")
else:
fh, imdfn = tempfile.mkstemp(suffix='.imd')
os.close(fh)
with codecs.open(imdfn, "w", encoding='utf-8') as file:
file.write(imd)
self.ui.mModelLineEdit.setText(imdfn)
def render(self):
""" do the rendering. This function is called in the worker thread """
debug("[WORKER THREAD] Calling request() asynchronously", 3)
QMetaObject.invokeMethod(self.controller, "request")
# setup a timer that checks whether the rendering has not been stopped
# in the meanwhile
timer = QTimer()
timer.setInterval(50)
timer.timeout.connect(self.onTimeout)
timer.start()
debug("[WORKER THREAD] Waiting for the async request to complete", 3)
self.loop = QEventLoop()
self.controller.finished.connect(self.loop.exit)
self.loop.exec_()
debug("[WORKER THREAD] Async request finished", 3)
painter = self.context.painter()
painter.drawImage(0, 0, self.controller.img)
return True
def get_topping_file_model(self, id_list):
topping_file_cache = IliToppingFileCache(
self.import_schema_configuration.base_configuration, id_list)
# we wait for the download or we timeout after 30 seconds and we apply what we have
loop = QEventLoop()
topping_file_cache.download_finished.connect(lambda: loop.quit())
timer = QTimer()
timer.setSingleShot(True)
timer.timeout.connect(lambda: loop.quit())
timer.start(30000)
topping_file_cache.refresh()
self.log_panel.print_info(self.tr("- - Downloading…"),
LogColor.COLOR_TOPPING)
if len(topping_file_cache.downloaded_files) != len(id_list):
loop.exec()
if len(topping_file_cache.downloaded_files) == len(id_list):
self.log_panel.print_info(
self.tr("- - All topping files successfully downloaded"),
LogColor.COLOR_TOPPING,
)
else:
missing_file_ids = id_list
for downloaded_file_id in topping_file_cache.downloaded_files:
if downloaded_file_id in missing_file_ids:
missing_file_ids.remove(downloaded_file_id)
self.log_panel.print_info(
self.
tr("- - Some topping files where not successfully downloaded: {}"
).format(" ".join(missing_file_ids)),
LogColor.COLOR_TOPPING,
)
return topping_file_cache.model
def testPythonCreateChildrenCalledFromCplusplus(self):
""" test createChildren() method implemented in Python, called from C++ """
loop = QEventLoop()
NUM_ITERS = 10 # set more to detect memory leaks
for i in range(NUM_ITERS):
tabSetDestroyedFlag = [False]
item = PyQgsDataConnectionItem(None, "name", "", "my_provider")
item.tabSetDestroyedFlag = tabSetDestroyedFlag
# Causes PyQgsDataConnectionItem.createChildren() to be called
item.populate()
# wait for populate() to have done its job
item.stateChanged.connect(loop.quit)
loop.exec_()
# Python object PyQgsLayerItem should still be alive
self.assertFalse(tabSetDestroyedFlag[0])
children = item.children()
self.assertEqual(len(children), 2)
self.assertEqual(children[0].name(), "name")
self.assertEqual(children[1].name(), "name2")
del (children)
# Delete the object and make sure all deferred deletions are processed
item.destroyed.connect(loop.quit)
item.deleteLater()
loop.exec_()
# Check that the PyQgsLayerItem Python object is now destroyed
self.assertTrue(tabSetDestroyedFlag[0])
tabSetDestroyedFlag[0] = False
def _sync_get(url):
global __network_manager
if __network_manager is None:
__network_manager = QNetworkAccessManager()
__network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
pause = QEventLoop()
req = QNetworkRequest(url)
req.setRawHeader(b"Accept", b"application/xml")
req.setRawHeader(b"Accept-Language",
bytes(settings.value("default_language", "fr"), "utf8"))
req.setRawHeader(
b"User-Agent",
bytes(settings.value("http_user_agent", plugin_name()), "utf8"))
reply = __network_manager.get(req)
reply.finished.connect(pause.quit)
is_ok = [True]
def onError(self):
is_ok[0] = False
pause.quit()
reply.error.connect(onError)
pause.exec_()
return reply, is_ok[0]
def upload_files(self, layer, field_index, features):
"""
Upload given features' source files to remote server and return a dict
formatted as changeAttributeValues expects to update 'datos' attribute
to a remote location.
"""
if not QSettings().value(
'Asistente-LADM_COL/sources/document_repository', False, bool):
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"The source files were not uploaded to the document repository because you have that option unchecked. You can still upload the source files later using the 'Upload Pending Source Files' menu."
), Qgis.Info, 10)
return dict()
# Test if we have Internet connection and a valid service
dlg = self.qgis_utils.get_settings_dialog()
res, msg = dlg.is_source_service_valid()
if not res:
msg['text'] = QCoreApplication.translate(
"SourceHandler",
"No file could be uploaded to the document repository. You can do it later from the 'Upload Pending Source Files' menu. Reason: {}"
).format(msg['text'])
self.message_with_duration_emitted.emit(
msg['text'], Qgis.Info,
20) # The data is still saved, so always show Info msg
return dict()
file_features = [
feature for feature in features if not feature[field_index] == NULL
and os.path.isfile(feature[field_index])
]
total = len(features)
not_found = total - len(file_features)
upload_dialog = UploadProgressDialog(len(file_features), not_found)
upload_dialog.show()
count = 0
upload_errors = 0
new_values = dict()
for feature in file_features:
data_url = feature[field_index]
file_name = os.path.basename(data_url)
nam = QNetworkAccessManager()
#reply.downloadProgress.connect(upload_dialog.update_current_progress)
multiPart = QHttpMultiPart(QHttpMultiPart.FormDataType)
textPart = QHttpPart()
textPart.setHeader(QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"driver\""))
textPart.setBody(QByteArray().append('Local'))
filePart = QHttpPart()
filePart.setHeader(
QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"file\"; filename=\"{}\"".format(
file_name)))
file = QFile(data_url)
file.open(QIODevice.ReadOnly)
filePart.setBodyDevice(file)
file.setParent(
multiPart
) # we cannot delete the file now, so delete it with the multiPart
multiPart.append(filePart)
multiPart.append(textPart)
service_url = '/'.join([
QSettings().value(
'Asistente-LADM_COL/sources/service_endpoint',
DEFAULT_ENDPOINT_SOURCE_SERVICE),
SOURCE_SERVICE_UPLOAD_SUFFIX
])
request = QNetworkRequest(QUrl(service_url))
reply = nam.post(request, multiPart)
#reply.uploadProgress.connect(upload_dialog.update_current_progress)
reply.error.connect(self.error_returned)
multiPart.setParent(reply)
# We'll block execution until we get response from the server
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
response = reply.readAll()
data = QTextStream(response, QIODevice.ReadOnly)
content = data.readAll()
if content is None:
self.log.logMessage(
"There was an error uploading file '{}'".format(data_url),
PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
try:
response = json.loads(content)
except json.decoder.JSONDecodeError:
self.log.logMessage(
"Couldn't parse JSON response from server for file '{}'!!!"
.format(data_url), PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
if 'error' in response:
self.log.logMessage(
"STATUS: {}. ERROR: {} MESSAGE: {} FILE: {}".format(
response['status'], response['error'],
response['message'], data_url), PLUGIN_NAME,
Qgis.Critical)
upload_errors += 1
continue
reply.deleteLater()
if 'url' not in response:
self.log.logMessage(
"'url' attribute not found in JSON response for file '{}'!"
.format(data_url), PLUGIN_NAME, Qgis.Critical)
upload_errors += 1
continue
url = self.get_file_url(response['url'])
new_values[feature.id()] = {field_index: url}
count += 1
upload_dialog.update_total_progress(count)
if not_found > 0:
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} records {} not uploaded to the document repository because {} file path is NULL or it couldn't be found in the local disk!"
).format(
not_found, total,
QCoreApplication.translate("SourceHandler", "was")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "their")), Qgis.Info, 0)
if len(new_values):
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files {} uploaded to the document repository and {} remote location stored in the database!"
).format(
len(new_values), total,
QCoreApplication.translate("SourceHandler", "was")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "their")), Qgis.Info, 0)
if upload_errors:
self.message_with_duration_emitted.emit(
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files could not be uploaded to the document repository because of upload errors! See log for details."
).format(upload_errors, total), Qgis.Info, 0)
return new_values
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'text': '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})
req = QNetworkRequest()
# Avoid double quoting form QUrl
if PYTHON_VERSION >= 3:
url = urllib.parse.unquote(url)
else:
url = urllib2.unquote(url)
req.setUrl(QUrl(url))
if self.cookie is not None:
if headers is not None:
headers['Cookie'] = self.cookie
else:
headers = {'Cookie': self.cookie}
if self.basicauth is not None:
if headers is not None:
headers['Authorization'] = self.basicauth
else:
headers = {'Authorization': self.basicauth}
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in headers.items():
if PYTHON_VERSION >= 3:
if isinstance(k, str):
k = k.encode('utf-8')
if isinstance(v, str):
v = v.encode('utf-8')
req.setRawHeader(k, v)
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if PYTHON_VERSION >= 3:
if isinstance(body, str):
body = body.encode('utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
QgsAuthManager.instance().updateNetworkReply(
self.reply, self.authid)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
self.reply.downloadProgress.connect(self.downloadProgress)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_()
# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
headers = {
str(h): str(self.reply.rawHeader(h))
for h in self.reply.rawHeaderList()
}
for k, v in headers.items():
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.text) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")
except Exception as e:
raise e
finally:
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.text)
def getVideoLocationInfo(videoPath, dataFile=False):
""" Get basic location info about the video """
location = []
try:
if dataFile == False:
p = _spawn([
'-i', videoPath, '-ss', '00:00:00', '-to', '00:00:01', '-map',
'data-re', '-f', 'data', '-'
])
stdout_data, _ = p.communicate()
################
else:
global listOfMetadata
stdout_data = listOfMetadata[0]
################
if stdout_data == b'':
return
for packet in StreamParser(stdout_data):
if isinstance(packet, UnknownElement):
qgsu.showUserAndLogMessage(
"Error interpreting klv data, metadata cannot be read.",
"the parser did not recognize KLV data",
level=QGis.Warning)
continue
packet.MetadataList()
frameCenterLat = packet.FrameCenterLatitude
frameCenterLon = packet.FrameCenterLongitude
loc = "-"
if Reverse_geocoding_url != "":
try:
url = QUrl(
Reverse_geocoding_url.format(str(frameCenterLat),
str(frameCenterLon)))
request = QNetworkRequest(url)
reply = QgsNetworkAccessManager.instance().get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
reply.finished.disconnect(loop.quit)
loop = None
result = reply.readAll()
data = json.loads(result.data())
if "village" in data["address"] and "state" in data[
"address"]:
loc = data["address"]["village"] + \
", " + data["address"]["state"]
elif "town" in data["address"] and "state" in data[
"address"]:
loc = data["address"]["town"] + \
", " + data["address"]["state"]
else:
loc = data["display_name"]
except Exception:
qgsu.showUserAndLogMessage(
"",
"getVideoLocationInfo: failed to get address from reverse geocoding service.",
onlyLog=True)
location = [frameCenterLat, frameCenterLon, loc]
qgsu.showUserAndLogMessage(
"",
"Got Location: lon: " + str(frameCenterLon) + " lat: " +
str(frameCenterLat) + " location: " + str(loc),
onlyLog=True)
break
else:
qgsu.showUserAndLogMessage(
QCoreApplication.translate(
"QgsFmvUtils", "This video doesn't have Metadata ! : "))
except Exception as e:
qgsu.showUserAndLogMessage(
QCoreApplication.translate("QgsFmvUtils",
"Video info callback failed! : "),
str(e))
return location
def run(self):
ili2db_bin = get_ili2db_bin(self.tool_name, self.stdout, self.stderr,
ili2db_tools)
if not ili2db_bin:
return
ili2db_jar_arg = ["-jar", ili2db_bin]
self.configuration.tool_name = self.tool_name
args = self.configuration.to_ili2db_args()
if self.dataImport:
args += [self.configuration.xtffile]
if self.configuration.base_configuration.java_path:
# A java path is configured: respect it no mather what
java_paths = [self.configuration.base_configuration.java_path]
else:
# By default try JAVA_HOME and PATH
java_paths = []
if 'JAVA_HOME' in os.environ:
paths = os.environ['JAVA_HOME'].split(os.pathsep)
for path in paths:
java_paths += [
os.path.join(
path.replace("\"", "").replace("'", ""), 'java')
]
java_paths += ['java']
proc = None
for java_path in java_paths:
proc = QProcess()
proc.readyReadStandardError.connect(
functools.partial(self.stderr_ready, proc=proc))
proc.readyReadStandardOutput.connect(
functools.partial(self.stdout_ready, proc=proc))
proc.start(java_path, ili2db_jar_arg + args)
if not proc.waitForStarted():
proc = None
else:
break
if not proc:
raise JavaNotFoundError()
safe_args = ili2db_jar_arg + self.configuration.to_ili2db_args(
hide_password=True)
safe_command = java_path + ' ' + ' '.join(safe_args)
self.process_started.emit(safe_command)
self.__result = Importer.ERROR
loop = QEventLoop()
proc.finished.connect(loop.exit)
loop.exec()
self.process_finished.emit(proc.exitCode(), self.__result)
return self.__result
def generate_report(self, db, button):
# Check if mapfish and Jasper are installed, otherwise show where to
# download them from and return
base_path = os.path.join(os.path.expanduser('~'), 'Asistente-LADM_COL',
'impresion')
bin_path = os.path.join(base_path, 'bin')
if not os.path.exists(bin_path):
self.qgis_utils.message_with_button_download_report_dependency_emitted.emit(
QCoreApplication.translate(
"ReportGenerator",
"The dependency library to generate reports is not installed. Click on the button to download and install it."
))
return
# Check version
required_version_found = True
version_path = os.path.join(base_path, 'version')
if not os.path.exists(version_path):
required_version_found = False
else:
version_found = ''
with open(version_path) as f:
version_found = f.read()
if version_found != REPORTS_REQUIRED_VERSION:
required_version_found = False
if not required_version_found:
self.qgis_utils.message_with_button_remove_report_dependency_emitted.emit(
QCoreApplication.translate(
"ReportGenerator",
"The dependency library to generate reports was found, but does not match with the version required. Click the button to remove the installed version and try again."
))
return
# Check if JAVA_HOME path is set, otherwise use path from project Generator
if os.name == 'nt':
if 'JAVA_HOME' not in os.environ:
java_path = self.get_java_path_from_project_generator()
if not java_path:
self.qgis_utils.message_emitted.emit(
QCoreApplication.translate(
"ReportGenerator",
"Please set JAVA_HOME path in Project Generator Settings or in Environmental Variables for your OS"
), Qgis.Warning)
return
else:
os.environ["JAVA_HOME"] = java_path
self.log.logMessage(
"The JAVA_HOME path has been set using Project Generator Settings for reports.",
PLUGIN_NAME, Qgis.Info)
plot_layer = self.qgis_utils.get_layer(db,
PLOT_TABLE,
QgsWkbTypes.PolygonGeometry,
load=True)
if plot_layer is None:
self.qgis_utils.message_emitted.emit(
QCoreApplication.translate(
"ReportGenerator",
"Layer 'Plot' not found in DB! {}").format(
db.get_description()), Qgis.Warning)
return
selected_plots = plot_layer.selectedFeatures()
if not selected_plots:
self.qgis_utils.message_emitted.emit(
QCoreApplication.translate(
"ReportGenerator",
"To generate reports, first select at least a plot!"),
Qgis.Warning)
return
# Where to store the reports?
previous_folder = QSettings().value(
"Asistente-LADM_COL/reports/save_into_dir", ".")
save_into_folder = QFileDialog.getExistingDirectory(
None,
QCoreApplication.translate(
"ReportGenerator",
"Select a folder to save the reports to be generated"),
previous_folder)
if not save_into_folder:
self.qgis_utils.message_emitted.emit(
QCoreApplication.translate(
"ReportGenerator",
"You need to select a folder where to save the reports before continuing."
), Qgis.Warning)
return
QSettings().setValue("Asistente-LADM_COL/reports/save_into_dir",
save_into_folder)
config_path = os.path.join(base_path, 'ANT')
json_spec_file = os.path.join(config_path, 'spec_json_file.json')
script_name = ''
if os.name == 'posix':
script_name = 'print'
elif os.name == 'nt':
script_name = 'print.bat'
script_path = os.path.join(bin_path, script_name)
if not os.path.isfile(script_path):
print("### SCRIPT FILE WASN'T FOUND")
return
button.setEnabled(False)
# Update config file
yaml_config_path = self.update_yaml_config(db, config_path)
print("CONFIG FILE:", yaml_config_path)
total = len(selected_plots)
step = 0
count = 0
tmp_dir = self.get_tmp_dir()
# Progress bar setup
progress = QProgressBar()
if total == 1:
progress.setRange(0, 0)
else:
progress.setRange(0, 100)
progress.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.qgis_utils.create_progress_message_bar_emitted.emit(
QCoreApplication.translate("ReportGenerator",
"Generating {} report{}...").format(
total, '' if total == 1 else 's'),
progress)
for selected_plot in selected_plots:
plot_id = selected_plot[ID_FIELD]
# Generate data file
json_file = self.update_json_data(db, json_spec_file, plot_id,
tmp_dir)
print("JSON FILE:", json_file)
# Run sh/bat passing config and data files
proc = QProcess()
proc.readyReadStandardError.connect(
functools.partial(self.stderr_ready, proc=proc))
proc.readyReadStandardOutput.connect(
functools.partial(self.stdout_ready, proc=proc))
current_report_path = os.path.join(
save_into_folder, 'anexo_17_{}.pdf'.format(plot_id))
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
# Grant execution permissions
os.chmod(
script_path, stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR
| stat.S_IRUSR | stat.S_IRGRP)
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
proc = None
print("### COULDN'T EXECUTE SCRIPT TO GENERATE REPORT...")
else:
loop = QEventLoop()
proc.finished.connect(loop.exit)
loop.exec()
print(plot_id, ':', proc.exitCode())
if proc.exitCode() == 0:
count += 1
step += 1
progress.setValue(step * 100 / total)
os.remove(yaml_config_path)
button.setEnabled(True)
self.qgis_utils.clear_message_bar_emitted.emit()
if total == count:
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report <a href='file://{}'>anexo_17_{}.pdf</a> was successfully generated!"
).format(save_into_folder, plot_id)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"All reports were successfully generated in folder <a href='file://{path}'>{path}</a>!"
).format(path=save_into_folder)
self.qgis_utils.message_with_duration_emitted.emit(
msg, Qgis.Success, 0)
else:
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report for plot {} couldn't be generated! See QGIS log (tab 'Anexo_17') for details."
).format(plot_id)
else:
if count == 0:
msg = QCoreApplication.translate(
"ReportGenerator",
"No report could be generated! See QGIS log (tab 'Anexo_17') for details."
)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"At least one report couldn't be generated! See QGIS log (tab 'Anexo_17') for details. Go to <a href='file://{path}'>{path}</a> to see the reports that were generated."
).format(path=save_into_folder)
self.qgis_utils.message_with_duration_emitted.emit(
msg, Qgis.Warning, 0)
def testStoreFetchFileLater(self):
"""
Test file storing and fetching (Later mode)
"""
f = self.getNewFile(b"New content")
# store
url = self.url + "/" + os.path.basename(f.name)
storedContent = self.storage.store(f.name, url, self.auth_config.id())
self.assertTrue(storedContent)
self.assertEqual(storedContent.status(), Qgis.ContentStatus.NotStarted)
spyErrorOccurred = QSignalSpy(storedContent.errorOccurred)
spyProgressChanged = QSignalSpy(storedContent.progressChanged)
loop = QEventLoop()
storedContent.stored.connect(loop.quit)
storedContent.errorOccurred.connect(loop.quit)
QTimer.singleShot(1, lambda: storedContent.store())
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(storedContent.url(), url)
self.assertFalse(storedContent.errorString())
self.assertEqual(storedContent.status(), Qgis.ContentStatus.Finished)
self.assertTrue(len(spyProgressChanged) > 0)
self.assertEqual(spyProgressChanged[-1][0], 100)
# fetch
fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id())
self.assertTrue(fetchedContent)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.NotStarted)
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.fetched.connect(loop.quit)
fetchedContent.errorOccurred.connect(loop.quit)
QTimer.singleShot(1, lambda: fetchedContent.fetch())
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
self.assertFalse(fetchedContent.errorString())
self.assertTrue(fetchedContent.filePath())
self.checkContent(fetchedContent.filePath(), b"New content")
self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')
# fetch again, should be cached
fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id())
self.assertTrue(fetchedContent)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.NotStarted)
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.fetched.connect(loop.quit)
fetchedContent.errorOccurred.connect(loop.quit)
QTimer.singleShot(1, lambda: fetchedContent.fetch())
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
self.assertTrue(not fetchedContent.errorString())
self.assertTrue(fetchedContent.filePath())
self.checkContent(fetchedContent.filePath(), b"New content")
self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')
# fetch bad url
fetchedContent = self.storage.fetch(self.url + "/error", self.auth_config.id())
self.assertTrue(fetchedContent)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.NotStarted)
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.fetched.connect(loop.quit)
fetchedContent.errorOccurred.connect(loop.quit)
QTimer.singleShot(1, lambda: fetchedContent.fetch())
loop.exec()
self.assertEqual(len(spyErrorOccurred), 1)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Failed)
self.assertTrue(fetchedContent.errorString())
self.assertFalse(fetchedContent.filePath())
def testStoreFetchFileImmediately(self):
"""
Test file storing and fetching (Immediately mode)
"""
f = self.getNewFile(b"New content")
# store
url = self.url + "/" + os.path.basename(f.name)
storedContent = self.storage.store(f.name, url, self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(storedContent)
self.assertEqual(storedContent.status(), Qgis.ContentStatus.Running)
spyErrorOccurred = QSignalSpy(storedContent.errorOccurred)
spyProgressChanged = QSignalSpy(storedContent.progressChanged)
loop = QEventLoop()
storedContent.stored.connect(loop.quit)
storedContent.errorOccurred.connect(loop.quit)
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(storedContent.url(), url)
self.assertFalse(storedContent.errorString())
self.assertEqual(storedContent.status(), Qgis.ContentStatus.Finished)
self.assertTrue(len(spyProgressChanged) > 0)
self.assertEqual(spyProgressChanged[-1][0], 100)
# fetch
fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(fetchedContent)
# Some external storage (SimpleCopy) doesn't actually need to retrieve the resource
self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Finished or
fetchedContent.status() == Qgis.ContentStatus.Running)
if (fetchedContent.status() == Qgis.ContentStatus.Running):
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.fetched.connect(loop.quit)
fetchedContent.errorOccurred.connect(loop.quit)
loop.exec()
self.assertEqual(len(spyErrorOccurred), 0)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
self.assertFalse(fetchedContent.errorString())
self.assertTrue(fetchedContent.filePath())
self.checkContent(fetchedContent.filePath(), b"New content")
self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')
# fetch again, should be cached
fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(fetchedContent)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished)
self.assertTrue(not fetchedContent.errorString())
self.assertTrue(fetchedContent.filePath())
self.checkContent(fetchedContent.filePath(), b"New content")
self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt')
# fetch bad url
fetchedContent = self.storage.fetch(self.url + "/error", self.auth_config.id(), Qgis.ActionStart.Immediate)
self.assertTrue(fetchedContent)
# Some external storage (SimpleCopy) doesn't actually need to retrieve the resource
self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Failed or
fetchedContent.status() == Qgis.ContentStatus.Running)
if (fetchedContent.status() == Qgis.ContentStatus.Running):
spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred)
loop = QEventLoop()
fetchedContent.errorOccurred.connect(loop.quit)
fetchedContent.fetched.connect(loop.quit)
loop.exec()
self.assertEqual(len(spyErrorOccurred), 1)
self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Failed)
self.assertTrue(fetchedContent.errorString())
self.assertFalse(fetchedContent.filePath())
def is_source_service_valid(self):
res = False
msg = {'text': '', 'level': Qgis.Warning}
url = self.txt_service_endpoint.text().strip()
if url:
with OverrideCursor(Qt.WaitCursor):
self.qgis_utils.status_bar_message_emitted.emit(
"Checking source service availability (this might take a while)...",
0)
QCoreApplication.processEvents()
if self.qgis_utils.is_connected(TEST_SERVER):
nam = QNetworkAccessManager()
request = QNetworkRequest(QUrl(url))
reply = nam.get(request)
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
allData = reply.readAll()
response = QTextStream(allData, QIODevice.ReadOnly)
status = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if status == 200:
try:
data = json.loads(response.readAll())
if 'id' in data and data[
'id'] == SOURCE_SERVICE_EXPECTED_ID:
res = True
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"The tested service is valid to upload files!"
)
msg['level'] = Qgis.Info
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"The tested upload service is not compatible: no valid 'id' found in response."
)
except json.decoder.JSONDecodeError as e:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"Response from the tested service is not compatible: not valid JSON found."
)
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to the server. The server might be down or the service cannot be reached at the given URL."
)
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog",
"There was a problem connecting to Internet.")
self.qgis_utils.clear_status_bar_emitted.emit()
else:
res = False
msg['text'] = QCoreApplication.translate(
"SettingsDialog", "Not valid service URL to test!")
return (res, msg)
def test(self):
# This test is quite fragile as it depends on windows manager behaviour
# regarding focus, so not surprising it doesn't pass
# on other platforms than Linux.
#if 'TRAVIS_OS_NAME' in os.environ and os.environ['TRAVIS_OS_NAME'] == 'osx':
# return
main_dialog = QgsProviderRegistry.instance().selectWidget("WFS")
main_dialog.setProperty("hideDialogs", True)
self.assertIsNotNone(main_dialog)
# Create new connection
btnNew = main_dialog.findChild(QWidget, "btnNew")
self.assertIsNotNone(btnNew)
QTest.mouseClick(btnNew, Qt.LeftButton)
new_conn = find_window('QgsNewHttpConnectionBase')
self.assertIsNotNone(new_conn)
txtName = new_conn.findChild(QLineEdit, "txtName")
self.assertIsNotNone(txtName)
txtName.setText("test_connection")
txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
self.assertIsNotNone(txtUrl)
txtUrl.setText("test_url")
new_conn.accept()
# Wait for object to be destroyed
new_conn = self.wait_object_destruction(new_conn)
# Try to connect
btnConnect = main_dialog.findChild(QWidget, "btnConnect")
self.assertIsNotNone(btnConnect)
QTest.mouseClick(btnConnect, Qt.LeftButton)
# Depends on asynchronous signal
QApplication.processEvents()
error_box = find_window('WFSCapabilitiesErrorBox')
self.assertIsNotNone(error_box)
# Close error box
error_box.accept()
# Wait for object to be destroyed
error_box = self.wait_object_destruction(error_box)
# Edit connection
btnEdit = main_dialog.findChild(QWidget, "btnEdit")
self.assertIsNotNone(btnEdit)
QTest.mouseClick(btnEdit, Qt.LeftButton)
new_conn = find_window('QgsNewHttpConnectionBase', )
self.assertIsNotNone(new_conn)
txtName = new_conn.findChild(QLineEdit, "txtName")
self.assertIsNotNone(txtName)
txtName.setText("test_connection")
txtUrl = new_conn.findChild(QLineEdit, "txtUrl")
self.assertIsNotNone(txtUrl)
endpoint = self.basetestpath + '/fake_qgis_http_endpoint'
expected_endpoint = endpoint
if sys.platform == 'win32' and expected_endpoint[1] == ':':
expected_endpoint = expected_endpoint[0] + expected_endpoint[2:]
with open(
sanitize(
endpoint,
'?SERVICE=WFS?REQUEST=GetCapabilities?ACCEPTVERSIONS=2.0.0,1.1.0,1.0.0'
), 'wb') as f:
f.write("""
<wfs:WFS_Capabilities version="2.0.0" xmlns="http://www.opengis.net/wfs/2.0" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://schemas.opengis.net/gml/3.2" xmlns:fes="http://www.opengis.net/fes/2.0">
<FeatureTypeList>
<FeatureType>
<Name>my:typename</Name>
<Title>Title</Title>
<Abstract>Abstract</Abstract>
<DefaultCRS>urn:ogc:def:crs:EPSG::4326</DefaultCRS>
<ows:WGS84BoundingBox>
<ows:LowerCorner>-71.123 66.33</ows:LowerCorner>
<ows:UpperCorner>-65.32 78.3</ows:UpperCorner>
</ows:WGS84BoundingBox>
</FeatureType>
</FeatureTypeList>
<fes:Filter_Capabilities>
<fes:Spatial_Capabilities>
<fes:GeometryOperands>
<fes:GeometryOperand name="gml:Envelope"/>
<fes:GeometryOperand name="gml:Point"/>
<fes:GeometryOperand name="gml:MultiPoint"/>
<fes:GeometryOperand name="gml:LineString"/>
<fes:GeometryOperand name="gml:MultiLineString"/>
<fes:GeometryOperand name="gml:Polygon"/>
<fes:GeometryOperand name="gml:MultiPolygon"/>
<fes:GeometryOperand name="gml:MultiGeometry"/>
</fes:GeometryOperands>
<fes:SpatialOperators>
<fes:SpatialOperator name="Disjoint"/>
<fes:SpatialOperator name="Equals"/>
<fes:SpatialOperator name="DWithin"/>
<fes:SpatialOperator name="Beyond"/>
<fes:SpatialOperator name="Intersects"/>
<fes:SpatialOperator name="Touches"/>
<fes:SpatialOperator name="Crosses"/>
<fes:SpatialOperator name="Within"/>
<fes:SpatialOperator name="Contains"/>
<fes:SpatialOperator name="Overlaps"/>
<fes:SpatialOperator name="BBOX"/>
</fes:SpatialOperators>
</fes:Spatial_Capabilities>
<fes:Functions>
<fes:Function name="abs">
<fes:Returns>xs:int</fes:Returns>
<fes:Arguments>
<fes:Argument name="param">
<fes:Type>xs:int</fes:Type>
</fes:Argument>
</fes:Arguments>
</fes:Function>
</fes:Functions>
</fes:Filter_Capabilities>
</wfs:WFS_Capabilities>""".encode('UTF-8'))
txtUrl.setText("http://" + endpoint)
new_conn.accept()
# Wait for object to be destroyed
new_conn = self.wait_object_destruction(new_conn)
# Try to connect
btnConnect = main_dialog.findChild(QWidget, "btnConnect")
self.assertIsNotNone(btnConnect)
QTest.mouseClick(btnConnect, Qt.LeftButton)
# We need to show (and raise for Mac) the dialog so that the focus changes
self.loop = QEventLoop()
treeView = main_dialog.findChild(QTreeView, "treeView")
treeView.selectionModel().currentRowChanged.connect(main_dialog.hide)
treeView.selectionModel().currentRowChanged.connect(self.loop.quit)
main_dialog.show()
main_dialog.raise_()
self.loop.exec_()
# Add layer
buttonAdd = self.get_button_add(main_dialog)
self.assertTrue(buttonAdd.isEnabled())
self.addWfsLayer_uri = None
self.addWfsLayer_layer_name = None
main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
QTest.mouseClick(buttonAdd, Qt.LeftButton)
self.assertEqual(
self.addWfsLayer_uri,
' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\''
+ "http://" + expected_endpoint +
'\' version=\'auto\' table="" sql=')
self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
# Click on Build Query
buttonBuildQuery = self.get_button_build_query(main_dialog)
self.assertTrue(buttonBuildQuery.isEnabled())
QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
error_box = find_window('WFSFeatureTypeErrorBox')
self.assertIsNotNone(error_box)
# Close error box
error_box.accept()
# Wait for object to be destroyed
error_box = self.wait_object_destruction(error_box)
# Click again but with valid DescribeFeatureType
with open(
sanitize(
endpoint,
'?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAME=my:typename'
), 'wb') as f:
f.write("""
<xsd:schema xmlns:my="http://my" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://my">
<xsd:import namespace="http://www.opengis.net/gml/3.2"/>
<xsd:complexType name="typenameType">
<xsd:complexContent>
<xsd:extension base="gml:AbstractFeatureType">
<xsd:sequence>
<xsd:element maxOccurs="1" minOccurs="0" name="intfield" nillable="true" type="xsd:int"/>
<xsd:element maxOccurs="1" minOccurs="0" name="geometryProperty" nillable="true" type="gml:PolygonPropertyType"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:element name="typename" substitutionGroup="gml:_Feature" type="my:typenameType"/>
</xsd:schema>
""".encode('UTF-8'))
QTest.mouseClick(buttonBuildQuery, Qt.LeftButton)
# Check that the combos are properly initialized
dialog = find_window('QgsSQLComposerDialogBase')
self.assertIsNotNone(dialog)
mTablesCombo = dialog.findChild(QComboBox, "mTablesCombo")
self.assertIsNotNone(mTablesCombo)
self.assertEqual(mTablesCombo.itemText(1), 'typename (Title)')
mColumnsCombo = dialog.findChild(QComboBox, "mColumnsCombo")
self.assertIsNotNone(mColumnsCombo)
self.assertEqual(mColumnsCombo.itemText(1), 'intfield (int)')
self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 2),
'geometryProperty (geometry)')
self.assertEqual(mColumnsCombo.itemText(mColumnsCombo.count() - 1),
'*')
mFunctionsCombo = dialog.findChild(QComboBox, "mFunctionsCombo")
self.assertIsNotNone(mFunctionsCombo)
self.assertEqual(mFunctionsCombo.itemText(1), 'abs(param: int): int')
mSpatialPredicatesCombo = dialog.findChild(QComboBox,
"mSpatialPredicatesCombo")
self.assertIsNotNone(mSpatialPredicatesCombo)
self.assertEqual(mSpatialPredicatesCombo.itemText(1),
'ST_Disjoint(geometry, geometry): boolean')
mWhereEditor = dialog.findChild(QTextEdit, "mWhereEditor")
self.assertIsNotNone(mWhereEditor)
mWhereEditor.setText('1 = 1')
dialog.accept()
# Wait for object to be destroyed
dialog = self.wait_object_destruction(dialog)
# Add layer
buttonAdd = self.get_button_add(main_dialog)
self.assertTrue(buttonAdd.isEnabled())
self.addWfsLayer_uri = None
self.addWfsLayer_layer_name = None
main_dialog.addWfsLayer.connect(self.slotAddWfsLayer)
QTest.mouseClick(buttonAdd, Qt.LeftButton)
self.assertEqual(
self.addWfsLayer_uri,
' retrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\''
+ "http://" + expected_endpoint +
'\' version=\'auto\' table="" sql=SELECT * FROM typename WHERE 1 = 1'
)
self.assertEqual(self.addWfsLayer_layer_name, 'my:typename')
from XYZHubConnector.xyz_qgis.network import NetManager
from XYZHubConnector.xyz_qgis.network import net_handler
from qgis.PyQt.QtCore import QEventLoop
from XYZHubConnector.xyz_qgis.models.connection import SpaceConnectionInfo
conn_info = SpaceConnectionInfo()
conn_info.set_(space_id="DicZ8XTR", token="AdOZrFlyIrXLzbAJeN5Lzts")
# minimize total number of features
# by generate tags for all levels
# smallest step for coord currently is 1 (can be smaller)
lst_tags = [format_tags(t, prefix="point") for t in precompute_tags()]
# print(lst_tags)
print("len tags", len(lst_tags))
tags_lst_obj = make_point_json(lst_tags)
print("len set tags", len(set(lst_tags)))
print("len tags_lst_obj", len(tags_lst_obj))
loop = QEventLoop()
network = NetManager(app)
total = sum(len(lst) for lst in tags_lst_obj.values())
print("total obj", total)
network.network.finished.connect(count_reply(total, loop.quit))
for i, (tags, lst_obj) in enumerate(tags_lst_obj.items()):
# print(len(lst_obj[0]["features"]), tags)
for obj in lst_obj:
pass
reply = network.add_features(conn_info, obj, tags=tags)
loop.exec_()
def generate_report(self, db, report_type):
# Check if mapfish and Jasper are installed, otherwise show where to
# download them from and return
if not self.report_dependency.check_if_dependency_is_valid():
self.report_dependency.download_dependency(URL_REPORTS_LIBRARIES)
return
java_home_set = self.java_dependency.set_java_home()
if not java_home_set:
self.java_dependency.get_java_on_demand()
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Java is a prerequisite. Since it was not found, it is being configured..."
))
return
plot_layer = self.app.core.get_layer(db, db.names.LC_PLOT_T, load=True)
if not plot_layer:
return
selected_plots = plot_layer.selectedFeatures()
if not selected_plots:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"To generate reports, first select at least a plot!"))
return
# Where to store the reports?
previous_folder = QSettings().value(
"Asistente-LADM-COL/reports/save_into_dir", ".")
save_into_folder = QFileDialog.getExistingDirectory(
None,
QCoreApplication.translate(
"ReportGenerator",
"Select a folder to save the reports to be generated"),
previous_folder)
if not save_into_folder:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"You need to select a folder where to save the reports before continuing."
))
return
QSettings().setValue("Asistente-LADM-COL/reports/save_into_dir",
save_into_folder)
config_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, report_type)
json_spec_file = os.path.join(config_path, 'spec_json_file.json')
script_name = ''
if os.name == 'posix':
script_name = 'print'
elif os.name == 'nt':
script_name = 'print.bat'
script_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, 'bin',
script_name)
if not os.path.isfile(script_path):
self.logger.warning(
__name__,
"Script file for reports wasn't found! {}".format(script_path))
return
self.enable_action_requested.emit(report_type, False)
# Update config file
yaml_config_path = self.update_yaml_config(db, config_path)
self.logger.debug(
__name__, "Config file for reports: {}".format(yaml_config_path))
total = len(selected_plots)
step = 0
count = 0
tmp_dir = self.get_tmp_dir()
# Progress bar setup
progress = QProgressBar()
if total == 1:
progress.setRange(0, 0)
else:
progress.setRange(0, 100)
progress.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.app.gui.create_progress_message_bar(
QCoreApplication.translate("ReportGenerator",
"Generating {} report{}...").format(
total, '' if total == 1 else 's'),
progress)
polygons_with_holes = []
multi_polygons = []
for selected_plot in selected_plots:
plot_id = selected_plot[db.names.T_ID_F]
geometry = selected_plot.geometry()
abstract_geometry = geometry.get()
if abstract_geometry.ringCount() > 1:
polygons_with_holes.append(str(plot_id))
self.logger.warning(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Skipping Annex 17 for plot with {}={} because it has holes. The reporter module does not support such polygons."
).format(db.names.T_ID_F, plot_id))
continue
if abstract_geometry.numGeometries() > 1:
multi_polygons.append(str(plot_id))
self.logger.warning(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Skipping Annex 17 for plot with {}={} because it is a multi-polygon. The reporter module does not support such polygons."
).format(db.names.T_ID_F, plot_id))
continue
# Generate data file
json_file = self.update_json_data(db, json_spec_file, plot_id,
tmp_dir, report_type)
self.logger.debug(__name__,
"JSON file for reports: {}".format(json_file))
# Run sh/bat passing config and data files
proc = QProcess()
proc.readyReadStandardError.connect(
functools.partial(self.stderr_ready, proc=proc))
proc.readyReadStandardOutput.connect(
functools.partial(self.stdout_ready, proc=proc))
parcel_number = self.ladm_data.get_parcels_related_to_plots(
db, [plot_id], db.names.LC_PARCEL_T_PARCEL_NUMBER_F) or ['']
file_name = '{}_{}_{}.pdf'.format(report_type, plot_id,
parcel_number[0])
current_report_path = os.path.join(save_into_folder, file_name)
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
# Grant execution permissions
os.chmod(
script_path, stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR
| stat.S_IRUSR | stat.S_IRGRP)
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
proc = None
self.logger.warning(
__name__, "Couldn't execute script to generate report...")
else:
loop = QEventLoop()
proc.finished.connect(loop.exit)
loop.exec()
self.logger.debug(__name__,
"{}:{}".format(plot_id, proc.exitCode()))
if proc.exitCode() == 0:
count += 1
step += 1
progress.setValue(step * 100 / total)
os.remove(yaml_config_path)
self.enable_action_requested.emit(report_type, True)
self.logger.clear_message_bar()
if total == count:
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report <a href='file:///{}'>{}</a> was successfully generated!"
).format(normalize_local_url(save_into_folder), file_name)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"All reports were successfully generated in folder <a href='file:///{path}'>{path}</a>!"
).format(path=normalize_local_url(save_into_folder))
self.logger.success_msg(__name__, msg)
else:
details_msg = ''
if polygons_with_holes:
details_msg += QCoreApplication.translate(
"ReportGenerator",
" The following polygons were skipped because they have holes and are not supported: {}."
).format(", ".join(polygons_with_holes))
if multi_polygons:
details_msg += QCoreApplication.translate(
"ReportGenerator",
" The following polygons were skipped because they are multi-polygons and are not supported: {}."
).format(", ".join(multi_polygons))
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report for plot {} couldn't be generated!{} See QGIS log (tab '{}') for details."
).format(plot_id, details_msg, self.LOG_TAB)
else:
if count == 0:
msg = QCoreApplication.translate(
"ReportGenerator",
"No report could be generated!{} See QGIS log (tab '{}') for details."
).format(details_msg, self.LOG_TAB)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"At least one report couldn't be generated!{details_msg} See QGIS log (tab '{log_tab}') for details. Go to <a href='file:///{path}'>{path}</a> to see the reports that were generated."
).format(details_msg=details_msg,
path=normalize_local_url(save_into_folder),
log_tab=self.LOG_TAB)
self.logger.warning_msg(__name__, msg)
def request(self,
url,
method="GET",
body=None,
headers=None,
redirections=DEFAULT_MAX_REDIRECTS,
connection_type=None,
blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))
self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
if k and v:
req.setRawHeader(k.encode(), v.encode())
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(),
'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" %
(method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, io.IOBase):
body = body.read()
if isinstance(body, str):
body = body.encode()
if isinstance(body, dict):
body = str(json.dumps(body)).encode(encoding='utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkReply(self.reply, self.authid)
QgsNetworkAccessManager.instance().setTimeout(int(self.timeout) * 1000)
# necessary to trap local timeout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(
self.requestTimedOut)
self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)
# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)
# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)
# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e
if self.reply:
self.reply.finished.disconnect(self.el.quit)
# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)
return (self.http_call_result, self.http_call_result.content)
def importgeopkg(self, layer, branch, message, authorName, authorEmail,
interchange):
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
filename, layername = namesFromLayer(layer)
r = requests.get(self.url + "beginTransaction",
params={"output_format": "json"})
r.raise_for_status()
transactionId = r.json()["response"]["Transaction"]["ID"]
self._checkoutbranch(branch, transactionId)
payload = {
"authorEmail": authorEmail,
"authorName": authorName,
"message": message,
'destPath': layername,
"format": "gpkg",
"transactionId": transactionId
}
# fix_print_with_import
if interchange:
payload["interchange"] = True
filename = self.saveaudittables(filename, layername)
files = {
'fileUpload': (os.path.basename(filename), open(filename, 'rb'))
}
encoder = MultipartEncoder(files)
total = float(encoder.len)
def callback(m):
done = int(100 * m.bytes_read / total)
iface.mainWindow().statusBar().showMessage(
"Transferring geopkg to GeoGig server [{}%]".format(done))
monitor = MultipartEncoderMonitor(encoder, callback)
r = requests.post(self.url + "import.json",
params=payload,
data=monitor,
headers={'Content-Type': monitor.content_type})
self.__log(r.url, r.text, payload, "POST")
r.raise_for_status()
resp = r.json()
taskId = resp["task"]["id"]
checker = TaskChecker(self.rootUrl, taskId)
loop = QEventLoop()
checker.taskIsFinished.connect(loop.exit, Qt.QueuedConnection)
checker.start()
loop.exec_(flags=QEventLoop.ExcludeUserInputEvents)
QApplication.restoreOverrideCursor()
iface.mainWindow().statusBar().showMessage("")
if not checker.ok and "error" in checker.response["task"]:
errorMessage = checker.response["task"]["error"]["message"]
raise GeoGigException("Cannot import layer: %s" % errorMessage)
if interchange:
try:
nconflicts = checker.response["task"]["result"]["Merge"][
"conflicts"]
except KeyError, e:
nconflicts = 0
if nconflicts:
mergeCommitId = self.HEAD
importCommitId = checker.response["task"]["result"]["import"][
"importCommit"]["id"]
ancestor = checker.response["task"]["result"]["Merge"][
"ancestor"]
remote = checker.response["task"]["result"]["Merge"]["ours"]
try:
featureIds = checker.response["task"]["result"]["import"][
"NewFeatures"]["type"][0].get("ids", [])
except:
featureIds = []
con = sqlite3.connect(filename)
cursor = con.cursor()
geomField = cursor.execute(
"SELECT column_name FROM gpkg_geometry_columns WHERE table_name='%s';"
% layername).fetchone()[0]
def _local(fid):
cursor.execute(
"SELECT gpkg_fid FROM %s_fids WHERE geogig_fid='%s';" %
(layername, fid))
gpkgfid = int(cursor.fetchone()[0])
request = QgsFeatureRequest()
request.setFilterFid(gpkgfid)
try:
feature = next(layer.getFeatures(request))
except:
return None
def _ensureNone(v):
if v == NULL:
return None
else:
return v
local = {
f.name(): _ensureNone(feature[f.name()])
for f in layer.pendingFields()
}
try:
local[geomField] = feature.geometry().exportToWkt()
except:
local[geomField] = None
return local
conflicts = []
conflictsResponse = _ensurelist(
checker.response["task"]["result"]["Merge"]["Feature"])
for c in conflictsResponse:
if c["change"] == "CONFLICT":
remoteFeatureId = c["ourvalue"]
localFeatureId = c["theirvalue"]
localFeature = _local(c["id"].split("/")[-1])
conflicts.append(
ConflictDiff(self, c["id"], ancestor, remote,
importCommitId, localFeature,
localFeatureId, remoteFeatureId,
transactionId))
cursor.close()
con.close()
else:
#self._checkoutbranch("master", transactionId)
self.closeTransaction(transactionId)
mergeCommitId = checker.response["task"]["result"][
"newCommit"]["id"]
importCommitId = checker.response["task"]["result"][
"importCommit"]["id"]
try:
featureIds = checker.response["task"]["result"][
"NewFeatures"]["type"][0].get("id", [])
except:
featureIds = []
conflicts = []
featureIds = [(f["provided"], f["assigned"]) for f in featureIds]
return mergeCommitId, importCommitId, conflicts, featureIds
